fix: deployment errors
This commit is contained in:
Rotimi Best
@@ -1,203 +1,98 @@
|
||||
import { eq, isNotNull } from 'drizzle-orm';
|
||||
import { and, eq, inArray, isNotNull, or, sql } from 'drizzle-orm';
|
||||
import { exercise, lesson } from '../schema';
|
||||
|
||||
import { db } from '../drizzle';
|
||||
import { sql } from 'drizzle-orm';
|
||||
import { mkdirSync, writeFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const scriptDir = path.dirname(fileURLToPath(import.meta.url));
|
||||
const backupDir = path.resolve(scriptDir, '../../backups');
|
||||
|
||||
const orderBatchSize = Number.parseInt(process.env.EXERCISE_BACKFILL_BATCH_SIZE ?? '200', 10) || 200;
|
||||
|
||||
/** Optional course scope: `pnpm db:exercise-backfill [courseId]` */
|
||||
const scopedCourseId = process.argv[2]?.trim() || null;
|
||||
|
||||
type LessonRow = {
|
||||
id: string;
|
||||
courseId: string;
|
||||
sectionId: string | null;
|
||||
order: number | null;
|
||||
createdAt: string | null;
|
||||
};
|
||||
|
||||
type ExerciseRow = {
|
||||
id: string;
|
||||
lessonId: string | null;
|
||||
courseId: string | null;
|
||||
sectionId: string | null;
|
||||
order: number | null;
|
||||
createdAt: string | null;
|
||||
};
|
||||
|
||||
type OrderUpdate = { id: string; order: number };
|
||||
|
||||
type FailedOrderBatch = {
|
||||
table: 'lesson' | 'exercise';
|
||||
batchIndex: number;
|
||||
rowCount: number;
|
||||
ids: string[];
|
||||
error: string;
|
||||
};
|
||||
|
||||
type OrderBackfillReport = {
|
||||
startedAt: string;
|
||||
completedAt: string;
|
||||
courseId: string | null;
|
||||
batchSize: number;
|
||||
summary: {
|
||||
lessonsAttempted: number;
|
||||
lessonsUpdated: number;
|
||||
lessonsFailed: number;
|
||||
exercisesAttempted: number;
|
||||
exercisesUpdated: number;
|
||||
exercisesFailed: number;
|
||||
};
|
||||
failures: FailedOrderBatch[];
|
||||
reportPath: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Migrates lesson-attached exercises from the V1 model to the V2 linear content model.
|
||||
*
|
||||
* **V1:** exercises nested under a lesson (`exercise.lesson_id`); lesson `order` was the
|
||||
* only sequence that mattered for navigation.
|
||||
*
|
||||
* **V2:** lessons and exercises share one `order` column per course (and per section when
|
||||
* `section_id` is set). The UI sorts both tables by that single sequence.
|
||||
*
|
||||
* Runs two phases in order:
|
||||
* 1. {@link setExerciseCourseAndSectionId} — copy `course_id` / `section_id` from the parent lesson.
|
||||
* 2. {@link assignLinearContentOrder} — renumber `lesson.order` and `exercise.order` in one shared sequence.
|
||||
*
|
||||
* @example
|
||||
* // Single course (dry-run scope while validating):
|
||||
* pnpm --filter @cio/db db:exercise-backfill 73432009-4c40-457d-9e90-6b48b18b0e23
|
||||
*
|
||||
* @example
|
||||
* // All courses:
|
||||
* pnpm --filter @cio/db db:exercise-backfill
|
||||
*/
|
||||
async function backfillExerciseCourse() {
|
||||
try {
|
||||
const result = await setExerciseCourseAndSectionId();
|
||||
const courseSectionResult = await setExerciseCourseAndSectionId(scopedCourseId);
|
||||
const orderResult = await assignLinearContentOrder(scopedCourseId);
|
||||
|
||||
// TypeScript implementation of exercise ordering logic
|
||||
// Algorithm:
|
||||
// 1. Fetch each lesson (grouped by course_id, section_id, ordered by lesson order or created_at)
|
||||
// 2. For each lesson, fetch its related exercises (ordered by createdAt)
|
||||
// 3. Create a linear sequence: lesson, then its exercises, then next lesson, etc.
|
||||
// 4. Assign sequential order numbers
|
||||
// 5. Update exercises with their new order values
|
||||
console.log('Exercise course/section backfill completed', courseSectionResult);
|
||||
console.log('Linear content order backfill completed', orderResult.summary);
|
||||
console.log('Report written to', orderResult.reportPath);
|
||||
|
||||
// Step 1: Fetch all lessons with their order (or calculate it from created_at)
|
||||
const allLessons = await db
|
||||
.select({
|
||||
id: lesson.id,
|
||||
courseId: lesson.courseId,
|
||||
sectionId: lesson.sectionId,
|
||||
order: lesson.order,
|
||||
createdAt: lesson.createdAt
|
||||
})
|
||||
.from(lesson);
|
||||
|
||||
// Group lessons by course_id and section_id
|
||||
type LessonWithOrder = (typeof allLessons)[0] & { calculatedOrder: number };
|
||||
const lessonsByGroup = new Map<string, LessonWithOrder[]>();
|
||||
|
||||
for (const lessonItem of allLessons) {
|
||||
const groupKey = `${lessonItem.courseId || 'null'}-${lessonItem.sectionId || 'null'}`;
|
||||
|
||||
if (!lessonsByGroup.has(groupKey)) {
|
||||
lessonsByGroup.set(groupKey, []);
|
||||
}
|
||||
// Add calculatedOrder property when pushing
|
||||
lessonsByGroup.get(groupKey)!.push({
|
||||
...lessonItem,
|
||||
calculatedOrder: lessonItem.order ?? 0 // Temporary, will be recalculated
|
||||
});
|
||||
if (orderResult.failures.length > 0) {
|
||||
console.error(`${orderResult.failures.length} batch(es) failed — see report for ids and errors.`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Calculate order for lessons that don't have one, within each group
|
||||
for (const [groupKey, groupLessons] of lessonsByGroup.entries()) {
|
||||
// Sort by existing order (nulls last), then by created_at
|
||||
groupLessons.sort((a, b) => {
|
||||
if (a.order !== null && b.order !== null) {
|
||||
return a.order - b.order;
|
||||
}
|
||||
if (a.order !== null) return -1;
|
||||
if (b.order !== null) return 1;
|
||||
return new Date(a.createdAt || '').getTime() - new Date(b.createdAt || '').getTime();
|
||||
});
|
||||
|
||||
// Assign calculated order
|
||||
groupLessons.forEach((lessonItem, index) => {
|
||||
lessonItem.calculatedOrder = lessonItem.order ?? index + 1;
|
||||
});
|
||||
}
|
||||
|
||||
// Step 2: Fetch all exercises that have a lesson_id
|
||||
const allExercises = await db
|
||||
.select({
|
||||
id: exercise.id,
|
||||
lessonId: exercise.lessonId,
|
||||
courseId: exercise.courseId,
|
||||
sectionId: exercise.sectionId,
|
||||
createdAt: exercise.createdAt
|
||||
})
|
||||
.from(exercise)
|
||||
.where(isNotNull(exercise.lessonId));
|
||||
|
||||
// Group exercises by lesson_id
|
||||
const exercisesByLesson = new Map<string, typeof allExercises>();
|
||||
for (const exerciseItem of allExercises) {
|
||||
if (!exerciseItem.lessonId) continue;
|
||||
|
||||
if (!exercisesByLesson.has(exerciseItem.lessonId)) {
|
||||
exercisesByLesson.set(exerciseItem.lessonId, []);
|
||||
}
|
||||
exercisesByLesson.get(exerciseItem.lessonId)!.push(exerciseItem);
|
||||
}
|
||||
|
||||
// Sort exercises within each lesson by createdAt
|
||||
for (const exercises of exercisesByLesson.values()) {
|
||||
exercises.sort((a, b) => {
|
||||
return new Date(a.createdAt || '').getTime() - new Date(b.createdAt || '').getTime();
|
||||
});
|
||||
}
|
||||
|
||||
// Step 3: Create linear sequence - lesson, then its exercises, then next lesson, etc.
|
||||
interface ContentItem {
|
||||
type: 'lesson' | 'exercise';
|
||||
lessonId: string | null;
|
||||
exerciseId: string | null;
|
||||
courseId: string;
|
||||
sectionId: string | null;
|
||||
lessonOrder: number;
|
||||
sortOffset: number;
|
||||
}
|
||||
|
||||
const allItems: ContentItem[] = [];
|
||||
|
||||
// Process each group (course_id + section_id combination)
|
||||
for (const [groupKey, groupLessons] of lessonsByGroup.entries()) {
|
||||
// Sort lessons by their calculated order
|
||||
const sortedLessons = [...groupLessons].sort((a, b) => a.calculatedOrder - b.calculatedOrder);
|
||||
|
||||
for (const lessonItem of sortedLessons) {
|
||||
// Add the lesson itself
|
||||
allItems.push({
|
||||
type: 'lesson',
|
||||
lessonId: lessonItem.id,
|
||||
exerciseId: null,
|
||||
courseId: lessonItem.courseId,
|
||||
sectionId: lessonItem.sectionId,
|
||||
lessonOrder: lessonItem.calculatedOrder,
|
||||
sortOffset: 0 // Lessons always have offset 0
|
||||
});
|
||||
|
||||
// Add exercises for this lesson (sorted by createdAt)
|
||||
const lessonExercises = exercisesByLesson.get(lessonItem.id) || [];
|
||||
for (let i = 0; i < lessonExercises.length; i++) {
|
||||
const exerciseItem = lessonExercises[i];
|
||||
allItems.push({
|
||||
type: 'exercise',
|
||||
lessonId: exerciseItem.lessonId,
|
||||
exerciseId: exerciseItem.id,
|
||||
courseId: exerciseItem.courseId || lessonItem.courseId,
|
||||
sectionId: exerciseItem.sectionId || lessonItem.sectionId,
|
||||
lessonOrder: lessonItem.calculatedOrder,
|
||||
sortOffset: i + 1 // Exercises have offset 1, 2, 3, etc.
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Step 4: Sort all items by lesson_order, then sort_offset (within each course_id + section_id group)
|
||||
allItems.sort((a, b) => {
|
||||
const groupKeyA = `${a.courseId}-${a.sectionId || 'null'}`;
|
||||
const groupKeyB = `${b.courseId}-${b.sectionId || 'null'}`;
|
||||
|
||||
if (groupKeyA !== groupKeyB) {
|
||||
return groupKeyA.localeCompare(groupKeyB);
|
||||
}
|
||||
|
||||
if (a.lessonOrder !== b.lessonOrder) {
|
||||
return a.lessonOrder - b.lessonOrder;
|
||||
}
|
||||
|
||||
return a.sortOffset - b.sortOffset;
|
||||
});
|
||||
|
||||
// Step 5: Assign sequential order numbers within each group and update exercises
|
||||
const orderUpdates: Array<{ exerciseId: string; order: number }> = [];
|
||||
let currentOrder = 1;
|
||||
let lastGroupKey = '';
|
||||
|
||||
for (const item of allItems) {
|
||||
const groupKey = `${item.courseId}-${item.sectionId || 'null'}`;
|
||||
|
||||
// Reset order counter when we move to a new group
|
||||
if (groupKey !== lastGroupKey) {
|
||||
currentOrder = 1;
|
||||
lastGroupKey = groupKey;
|
||||
}
|
||||
|
||||
// Only update exercises (not lessons)
|
||||
if (item.type === 'exercise' && item.exerciseId) {
|
||||
orderUpdates.push({
|
||||
exerciseId: item.exerciseId,
|
||||
order: currentOrder
|
||||
});
|
||||
}
|
||||
|
||||
currentOrder++;
|
||||
}
|
||||
|
||||
// Batch update exercises
|
||||
let exerciseOrderResult;
|
||||
if (orderUpdates.length > 0) {
|
||||
// Use a transaction to update all exercises
|
||||
exerciseOrderResult = await db.transaction(async (tx) => {
|
||||
const updatePromises = orderUpdates.map((update) =>
|
||||
tx.update(exercise).set({ order: update.order }).where(eq(exercise.id, update.exerciseId))
|
||||
);
|
||||
await Promise.all(updatePromises);
|
||||
return { rowCount: orderUpdates.length };
|
||||
});
|
||||
} else {
|
||||
exerciseOrderResult = { rowCount: 0 };
|
||||
}
|
||||
|
||||
console.log('Exercise backfill completed', result);
|
||||
console.log('Exercise order backfill completed', exerciseOrderResult);
|
||||
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
console.error('Failed to backfill exercise course data', error);
|
||||
@@ -205,7 +100,154 @@ async function backfillExerciseCourse() {
|
||||
}
|
||||
}
|
||||
|
||||
async function setExerciseCourseAndSectionId() {
|
||||
/**
|
||||
* Assigns a single 1-based `order` sequence to lessons and their child exercises.
|
||||
*
|
||||
* **Grouping** — Items are processed independently per `(course_id, section_id)` bucket.
|
||||
* Unsectioned content uses `section_id = null`. Section order is not changed here.
|
||||
*
|
||||
* **Lesson ordering** — Within each bucket, lessons are sorted by existing `order`
|
||||
* (ascending), with `null` orders last; ties break on `created_at`.
|
||||
*
|
||||
* **Exercise ordering** — Exercises with a `lesson_id` are grouped under that lesson.
|
||||
* Within a lesson, exercises sort by `order` then `created_at` (same comparator as lessons).
|
||||
*
|
||||
* **Linear sequence** — For each lesson in sorted order, emit one slot for the lesson,
|
||||
* then one slot per child exercise (in exercise sort order). Assign `order = 1..n` across
|
||||
* that combined list and persist both tables in batched `CASE` updates
|
||||
* (`EXERCISE_BACKFILL_BATCH_SIZE`, default 200). Failed batches are logged and skipped;
|
||||
* the run continues until all batches are attempted. See `packages/db/backups/exercise-backfill-report-*.json`.
|
||||
*
|
||||
* Example (one section bucket):
|
||||
*
|
||||
* ```
|
||||
* Before (lesson order only): After (shared order):
|
||||
* L1 order=1 L1 order=1
|
||||
* └ Ea (nested) Ea order=2
|
||||
* └ Eb Eb order=3
|
||||
* L2 order=2 L2 order=4
|
||||
* └ Ec Ec order=5
|
||||
* ```
|
||||
*
|
||||
* Only exercises that still have `lesson_id` are included. Course-level exercises
|
||||
* (`lesson_id IS NULL`) are left unchanged.
|
||||
*
|
||||
* @param courseId — When set, only lessons/exercises for this course are loaded and updated.
|
||||
*/
|
||||
async function assignLinearContentOrder(courseId: string | null) {
|
||||
const lessonWhere = courseId ? eq(lesson.courseId, courseId) : undefined;
|
||||
|
||||
const allLessons = await db
|
||||
.select({
|
||||
id: lesson.id,
|
||||
courseId: lesson.courseId,
|
||||
sectionId: lesson.sectionId,
|
||||
order: lesson.order,
|
||||
createdAt: lesson.createdAt
|
||||
})
|
||||
.from(lesson)
|
||||
.where(lessonWhere);
|
||||
|
||||
const allExercises = await db
|
||||
.select({
|
||||
id: exercise.id,
|
||||
lessonId: exercise.lessonId,
|
||||
courseId: exercise.courseId,
|
||||
sectionId: exercise.sectionId,
|
||||
order: exercise.order,
|
||||
createdAt: exercise.createdAt
|
||||
})
|
||||
.from(exercise)
|
||||
.leftJoin(lesson, eq(exercise.lessonId, lesson.id))
|
||||
.where(
|
||||
courseId
|
||||
? and(or(eq(exercise.courseId, courseId), eq(lesson.courseId, courseId)), isNotNull(exercise.lessonId))
|
||||
: isNotNull(exercise.lessonId)
|
||||
);
|
||||
|
||||
const lessonsByGroup = new Map<string, LessonRow[]>();
|
||||
for (const lessonRow of allLessons) {
|
||||
const key = groupKey(lessonRow.courseId, lessonRow.sectionId);
|
||||
const group = lessonsByGroup.get(key) ?? [];
|
||||
group.push(lessonRow);
|
||||
lessonsByGroup.set(key, group);
|
||||
}
|
||||
|
||||
const exercisesByLesson = new Map<string, ExerciseRow[]>();
|
||||
for (const exerciseRow of allExercises) {
|
||||
if (!exerciseRow.lessonId) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const lessonExercises = exercisesByLesson.get(exerciseRow.lessonId) ?? [];
|
||||
lessonExercises.push(exerciseRow);
|
||||
exercisesByLesson.set(exerciseRow.lessonId, lessonExercises);
|
||||
}
|
||||
|
||||
for (const lessonExercises of exercisesByLesson.values()) {
|
||||
sortExercises(lessonExercises);
|
||||
}
|
||||
|
||||
const lessonOrderUpdates: Array<{ id: string; order: number }> = [];
|
||||
const exerciseOrderUpdates: Array<{ id: string; order: number }> = [];
|
||||
|
||||
for (const [, groupLessons] of lessonsByGroup) {
|
||||
const sortedLessons = sortLessons(groupLessons);
|
||||
let nextOrder = 1;
|
||||
|
||||
for (const lessonRow of sortedLessons) {
|
||||
lessonOrderUpdates.push({ id: lessonRow.id, order: nextOrder });
|
||||
nextOrder += 1;
|
||||
|
||||
const lessonExercises = exercisesByLesson.get(lessonRow.id) ?? [];
|
||||
for (const exerciseRow of lessonExercises) {
|
||||
exerciseOrderUpdates.push({ id: exerciseRow.id, order: nextOrder });
|
||||
nextOrder += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const startedAt = new Date().toISOString();
|
||||
const failures: FailedOrderBatch[] = [];
|
||||
|
||||
const lessonsUpdated = await applyBatchedOrderUpdates('lesson', lessonOrderUpdates, failures);
|
||||
const exercisesUpdated = await applyBatchedOrderUpdates('exercise', exerciseOrderUpdates, failures);
|
||||
|
||||
const report = buildOrderBackfillReport({
|
||||
startedAt,
|
||||
courseId,
|
||||
lessonOrderUpdates,
|
||||
exerciseOrderUpdates,
|
||||
lessonsUpdated,
|
||||
exercisesUpdated,
|
||||
failures
|
||||
});
|
||||
|
||||
mkdirSync(backupDir, { recursive: true });
|
||||
writeFileSync(report.reportPath, JSON.stringify(report, null, 2));
|
||||
|
||||
return report;
|
||||
}
|
||||
|
||||
/**
|
||||
* Denormalizes `course_id` and `section_id` onto exercises from their parent lesson.
|
||||
*
|
||||
* Updates rows where `lesson_id` is set and either `course_id` or `section_id` is still null,
|
||||
* so later queries can filter exercises by course without joining `lesson`.
|
||||
*/
|
||||
async function setExerciseCourseAndSectionId(courseId: string | null) {
|
||||
if (courseId) {
|
||||
return db.execute(sql`
|
||||
UPDATE exercise AS e
|
||||
SET course_id = l.course_id,
|
||||
section_id = l.section_id
|
||||
FROM lesson AS l
|
||||
WHERE e.lesson_id = l.id
|
||||
AND l.course_id = ${courseId}
|
||||
AND (e.course_id IS NULL OR e.section_id IS NULL)
|
||||
`);
|
||||
}
|
||||
|
||||
return db.execute(sql`
|
||||
UPDATE exercise AS e
|
||||
SET course_id = l.course_id,
|
||||
@@ -216,4 +258,131 @@ async function setExerciseCourseAndSectionId() {
|
||||
`);
|
||||
}
|
||||
|
||||
async function applyBatchedOrderUpdates(
|
||||
table: 'lesson' | 'exercise',
|
||||
updates: OrderUpdate[],
|
||||
failures: FailedOrderBatch[]
|
||||
) {
|
||||
if (updates.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let updatedCount = 0;
|
||||
const totalBatches = Math.ceil(updates.length / orderBatchSize);
|
||||
|
||||
for (let batchIndex = 0; batchIndex < totalBatches; batchIndex += 1) {
|
||||
const batchStart = batchIndex * orderBatchSize;
|
||||
const batch = updates.slice(batchStart, batchStart + orderBatchSize);
|
||||
const batchIds = batch.map((update) => update.id);
|
||||
|
||||
try {
|
||||
if (table === 'lesson') {
|
||||
await db
|
||||
.update(lesson)
|
||||
.set({
|
||||
order: sql`case ${lesson.id} ${sql.join(
|
||||
batch.map((update) => sql`when ${update.id} then ${update.order}`),
|
||||
sql` `
|
||||
)} else ${lesson.order} end`
|
||||
})
|
||||
.where(inArray(lesson.id, batchIds));
|
||||
} else {
|
||||
await db
|
||||
.update(exercise)
|
||||
.set({
|
||||
order: sql`case ${exercise.id} ${sql.join(
|
||||
batch.map((update) => sql`when ${update.id} then ${update.order}`),
|
||||
sql` `
|
||||
)} else ${exercise.order} end`
|
||||
})
|
||||
.where(inArray(exercise.id, batchIds));
|
||||
}
|
||||
|
||||
updatedCount += batch.length;
|
||||
console.log(`Updated ${table} batch ${batchIndex + 1}/${totalBatches} (${batch.length} rows)`);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
|
||||
failures.push({
|
||||
table,
|
||||
batchIndex,
|
||||
rowCount: batch.length,
|
||||
ids: batchIds,
|
||||
error: message
|
||||
});
|
||||
|
||||
console.error(`Failed ${table} batch ${batchIndex + 1}/${totalBatches} (${batch.length} rows):`, message);
|
||||
}
|
||||
}
|
||||
|
||||
return updatedCount;
|
||||
}
|
||||
|
||||
function buildOrderBackfillReport(input: {
|
||||
startedAt: string;
|
||||
courseId: string | null;
|
||||
lessonOrderUpdates: OrderUpdate[];
|
||||
exerciseOrderUpdates: OrderUpdate[];
|
||||
lessonsUpdated: number;
|
||||
exercisesUpdated: number;
|
||||
failures: FailedOrderBatch[];
|
||||
}): OrderBackfillReport {
|
||||
const stamp = input.startedAt.replace(/[:.]/g, '-');
|
||||
const reportPath = path.join(backupDir, `exercise-backfill-report-${stamp}.json`);
|
||||
|
||||
const lessonsAttempted = input.lessonOrderUpdates.length;
|
||||
const exercisesAttempted = input.exerciseOrderUpdates.length;
|
||||
const lessonsFailed = input.failures
|
||||
.filter((failure) => failure.table === 'lesson')
|
||||
.reduce((count, failure) => count + failure.rowCount, 0);
|
||||
const exercisesFailed = input.failures
|
||||
.filter((failure) => failure.table === 'exercise')
|
||||
.reduce((count, failure) => count + failure.rowCount, 0);
|
||||
|
||||
return {
|
||||
startedAt: input.startedAt,
|
||||
completedAt: new Date().toISOString(),
|
||||
courseId: input.courseId,
|
||||
batchSize: orderBatchSize,
|
||||
summary: {
|
||||
lessonsAttempted,
|
||||
lessonsUpdated: input.lessonsUpdated,
|
||||
lessonsFailed,
|
||||
exercisesAttempted,
|
||||
exercisesUpdated: input.exercisesUpdated,
|
||||
exercisesFailed
|
||||
},
|
||||
failures: input.failures,
|
||||
reportPath
|
||||
};
|
||||
}
|
||||
|
||||
function groupKey(courseId: string, sectionId: string | null) {
|
||||
return `${courseId}-${sectionId ?? 'null'}`;
|
||||
}
|
||||
|
||||
function compareByOrderThenCreatedAt<T extends { order: number | null; createdAt: string | null }>(left: T, right: T) {
|
||||
if (left.order !== null && right.order !== null) {
|
||||
return left.order - right.order;
|
||||
}
|
||||
|
||||
if (left.order !== null) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (right.order !== null) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
return new Date(left.createdAt || '').getTime() - new Date(right.createdAt || '').getTime();
|
||||
}
|
||||
|
||||
function sortLessons(lessons: LessonRow[]) {
|
||||
return [...lessons].sort(compareByOrderThenCreatedAt);
|
||||
}
|
||||
|
||||
function sortExercises(exercises: ExerciseRow[]) {
|
||||
return [...exercises].sort(compareByOrderThenCreatedAt);
|
||||
}
|
||||
|
||||
backfillExerciseCourse();
|
||||
|
||||
@@ -0,0 +1,348 @@
|
||||
/**
|
||||
* Reads **`course`** table data from a **legacy** Postgres dump (before `course.version` was dropped)
|
||||
* and disables content grouping for former **V1** courses on the live DB (`DATABASE_URL`).
|
||||
*
|
||||
* Legacy model (see commented `0000_stale_storm.sql` and `DROP COLUMN "version"` in
|
||||
* `0003_fancy_harry_osborn.sql`):
|
||||
* - Column **`version`** had type **`"COURSE_VERSION"`** enum with values **`V1`** and **`V2`**.
|
||||
*
|
||||
* App stores grouping in **`course.metadata` → `isContentGroupingEnabled`**. This script sets
|
||||
* **`metadata.isContentGroupingEnabled = false`** for every course id whose dump row has **`version = V1`**.
|
||||
* Rows with **`V2`** or other values are unchanged.
|
||||
*
|
||||
* **Your dump must still contain the `version` column** (pg_dump from before that migration, or a
|
||||
* full schema+data backup from that era). Dumps taken after `version` was dropped cannot be used—
|
||||
* the script exits with a clear error if `COPY ... course (...)` has no `version` column.
|
||||
*
|
||||
* **Plain SQL** (`.sql`): finds `COPY public.course` / `COPY course` with column list, parses tab rows.
|
||||
* **Custom** (`.dump`): `pg_restore --data-only -t course`.
|
||||
*
|
||||
* Limitation: COPY rows are split on **tab** boundaries. If a text/json column contains a literal tab,
|
||||
* parsing may mis-align; use a plain-format dump from pg_dump defaults (usually fine).
|
||||
*
|
||||
* Usage:
|
||||
* DATABASE_URL="..." pnpm exec tsx src/scripts/disable-v1-content-grouping-from-dump.ts path/to/backup.sql
|
||||
* pnpm exec tsx src/scripts/disable-v1-content-grouping-from-dump.ts --dry-run path/to/backup.dump
|
||||
*
|
||||
* Env: `DATABASE_URL`, `PG_RESTORE_BIN`, `PSQL_BIN` (same as other restore scripts).
|
||||
*/
|
||||
|
||||
import 'dotenv/config';
|
||||
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const scriptDir = path.dirname(fileURLToPath(import.meta.url));
|
||||
const backupDir = path.resolve(scriptDir, '../../backups');
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const dryRun = args.includes('--dry-run');
|
||||
const dumpPathRaw = args.find((argument) => !argument.startsWith('-'));
|
||||
|
||||
const databaseUrl = process.env.DATABASE_URL;
|
||||
const pgRestoreExecutable = process.env.PG_RESTORE_BIN ?? 'pg_restore';
|
||||
const psqlExecutable = process.env.PSQL_BIN ?? 'psql';
|
||||
|
||||
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
|
||||
function slicePlainCopyCourse(fileText: string) {
|
||||
const copyPublic = /\bCOPY\s+public\.course\b/i.exec(fileText);
|
||||
const copyBare = /\bCOPY\s+course\b/i.exec(fileText);
|
||||
|
||||
const matchIndex = copyPublic?.index ?? copyBare?.index;
|
||||
if (matchIndex === undefined) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const fromStdinIdx = fileText.indexOf('FROM stdin', matchIndex);
|
||||
if (fromStdinIdx === -1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let lineEnd = fileText.indexOf('\n', fromStdinIdx);
|
||||
if (lineEnd === -1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const terminator = /\n\\\.\s*(?:\n|\r?\n|$)/;
|
||||
const dataRegion = fileText.slice(lineEnd);
|
||||
const terminatorMatch = terminator.exec(dataRegion);
|
||||
if (!terminatorMatch) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const terminatorEndAbs = lineEnd + terminatorMatch.index + terminatorMatch[0].length;
|
||||
|
||||
return fileText.slice(matchIndex, terminatorEndAbs);
|
||||
}
|
||||
|
||||
function pgRestoreCourseExtract(dumpFilePath: string) {
|
||||
const tempDir = mkdtempSync(path.join(tmpdir(), 'v1-course-copy-'));
|
||||
const extractedPath = path.join(tempDir, 'course.sql');
|
||||
|
||||
try {
|
||||
const flags = spawnSync(
|
||||
pgRestoreExecutable,
|
||||
[
|
||||
'--data-only',
|
||||
'--no-owner',
|
||||
'--no-privileges',
|
||||
'-n',
|
||||
'public',
|
||||
'-t',
|
||||
'course',
|
||||
'-f',
|
||||
extractedPath,
|
||||
dumpFilePath
|
||||
],
|
||||
{ encoding: 'utf8' }
|
||||
);
|
||||
|
||||
if (flags.status !== 0) {
|
||||
console.error(pgRestoreExecutable, '--data-only extraction failed:', (flags.stderr ?? '').trim() || flags.status);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
return readFileSync(extractedPath, 'utf8');
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Parse `COPY ... course ( a, b, "c" ) FROM stdin;` */
|
||||
function parseCopyColumnList(copyBlock: string): string[] | null {
|
||||
const lineEnd = copyBlock.indexOf('\n');
|
||||
const firstLine = lineEnd === -1 ? copyBlock : copyBlock.slice(0, lineEnd);
|
||||
|
||||
const withCols = /COPY\s+(?:public\.)?course\s*\(\s*([^)]+)\s*\)\s*FROM\s+stdin\s*;?\s*$/im.exec(firstLine.trim());
|
||||
|
||||
if (!withCols) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const raw = withCols[1];
|
||||
|
||||
const columns: string[] = [];
|
||||
let current = '';
|
||||
let inQuote = false;
|
||||
|
||||
for (let index = 0; index < raw.length; index++) {
|
||||
const character = raw[index];
|
||||
|
||||
if (character === '"') {
|
||||
inQuote = !inQuote;
|
||||
current += character;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (character === ',' && !inQuote) {
|
||||
columns.push(stripIdent(current.trim()));
|
||||
current = '';
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
current += character;
|
||||
}
|
||||
|
||||
if (current.trim()) {
|
||||
columns.push(stripIdent(current.trim()));
|
||||
}
|
||||
|
||||
return columns;
|
||||
}
|
||||
|
||||
function stripIdent(identifier: string) {
|
||||
return identifier.replace(/^"|"$/g, '');
|
||||
}
|
||||
|
||||
function extractV1CourseIdsFromCopyBlock(copyBlock: string): string[] | null {
|
||||
const columns = parseCopyColumnList(copyBlock);
|
||||
|
||||
if (!columns?.length) {
|
||||
console.error(
|
||||
'Could not parse COPY column list for course. Use a dump where pg_dump included explicit columns, e.g. `COPY public.course (id, …, version, …) FROM stdin;`'
|
||||
);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
const idIndex = columns.findIndex((columnName) => columnName === 'id');
|
||||
const versionIndex = columns.findIndex((columnName) => columnName === 'version');
|
||||
|
||||
if (idIndex === -1) {
|
||||
console.error('COPY column list for course has no `id` column.');
|
||||
return null;
|
||||
}
|
||||
|
||||
if (versionIndex === -1) {
|
||||
console.error(
|
||||
'COPY column list for course has no `version` column. This script needs a **legacy** dump from before `course.version` was dropped (migration 0003).'
|
||||
);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
const newlineAfterHeader = copyBlock.indexOf('\n');
|
||||
const dataPart = newlineAfterHeader === -1 ? '' : copyBlock.slice(newlineAfterHeader + 1);
|
||||
const lines = dataPart.split(/\r?\n/);
|
||||
|
||||
const v1Ids = new Set<string>();
|
||||
|
||||
for (const line of lines) {
|
||||
if (line === '\\.') {
|
||||
break;
|
||||
}
|
||||
|
||||
if (line === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
/** Postgres COPY text format: tab-separated; `\N` is null */
|
||||
const fields = line.split('\t');
|
||||
if (fields.length < Math.max(idIndex, versionIndex) + 1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const rawId = fields[idIndex]?.trim() ?? '';
|
||||
const rawVersion = fields[versionIndex]?.trim() ?? '';
|
||||
|
||||
if (rawVersion === '\\N' || rawVersion === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
const versionNormalized = rawVersion.replace(/^"|"$/g, '').toUpperCase();
|
||||
|
||||
if (versionNormalized !== 'V1') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!UUID_RE.test(rawId)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
v1Ids.add(rawId);
|
||||
}
|
||||
|
||||
return [...v1Ids];
|
||||
}
|
||||
|
||||
function buildUpdateSql(courseIds: string[]) {
|
||||
const list = courseIds.map((courseId) => `'${courseId.replace(/'/g, "''")}'`).join(',\n ');
|
||||
|
||||
return `-- Former COURSE_VERSION = V1 → metadata.isContentGroupingEnabled = false
|
||||
BEGIN;
|
||||
|
||||
UPDATE public.course
|
||||
SET metadata = jsonb_set(
|
||||
COALESCE(metadata, '{}'::jsonb),
|
||||
'{isContentGroupingEnabled}',
|
||||
'false'::jsonb,
|
||||
true
|
||||
)
|
||||
WHERE id IN (
|
||||
${list}
|
||||
);
|
||||
|
||||
COMMIT;
|
||||
`;
|
||||
}
|
||||
|
||||
function printUsage() {
|
||||
console.error(`Usage:
|
||||
DATABASE_URL="postgres://..." pnpm exec tsx src/scripts/disable-v1-content-grouping-from-dump.ts <dump.dump|.sql>
|
||||
|
||||
Flags:
|
||||
--dry-run Write SQL to packages/db/backups/disable-v1-content-grouping-preview.sql only`);
|
||||
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
function main() {
|
||||
if (!dumpPathRaw) {
|
||||
console.error('Missing dump file path.');
|
||||
printUsage();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (!dryRun && !databaseUrl) {
|
||||
console.error('Set DATABASE_URL (or use --dry-run).');
|
||||
printUsage();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const resolvedDump = path.isAbsolute(dumpPathRaw) ? dumpPathRaw : path.resolve(process.cwd(), dumpPathRaw);
|
||||
|
||||
let copyBlock: string | null;
|
||||
|
||||
if (resolvedDump.endsWith('.sql')) {
|
||||
const text = readFileSync(resolvedDump, 'utf8');
|
||||
copyBlock = slicePlainCopyCourse(text);
|
||||
} else {
|
||||
const extracted = pgRestoreCourseExtract(resolvedDump);
|
||||
copyBlock = extracted ? slicePlainCopyCourse(extracted) : null;
|
||||
}
|
||||
|
||||
if (!copyBlock) {
|
||||
console.error('Could not find COPY block for public.course. Try: pg_restore -l your.dump | grep course');
|
||||
process.exitCode = 1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const v1Ids = extractV1CourseIdsFromCopyBlock(copyBlock);
|
||||
|
||||
if (v1Ids === null) {
|
||||
process.exitCode = 1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (v1Ids.length === 0) {
|
||||
console.log('No courses with version V1 found in dump (or no valid rows). Nothing to update.');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Found ${v1Ids.length} course id(s) with legacy version V1 in dump.`);
|
||||
|
||||
const sql = buildUpdateSql(v1Ids);
|
||||
|
||||
mkdirSync(backupDir, { recursive: true });
|
||||
const previewFile = path.join(backupDir, 'disable-v1-content-grouping-preview.sql');
|
||||
writeFileSync(previewFile, sql, 'utf8');
|
||||
console.log('Wrote SQL:', previewFile);
|
||||
|
||||
if (dryRun) {
|
||||
console.log('Dry run: not executing psql.');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const invoke = spawnSync(psqlExecutable, [databaseUrl!, '-v', 'ON_ERROR_STOP=1', '-f', previewFile], {
|
||||
stdio: 'inherit'
|
||||
});
|
||||
|
||||
if (invoke.error) {
|
||||
console.error('psql spawn failed:', invoke.error.message);
|
||||
process.exitCode = 1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (invoke.status !== 0) {
|
||||
process.exitCode = invoke.status ?? 1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('Done: metadata.isContentGroupingEnabled set to false for V1 courses (where ids exist).');
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* Full logical backup of the database into `packages/db/backups/` using `pg_dump`.
|
||||
*
|
||||
* Requires PostgreSQL client tools (`pg_dump`) on PATH unless `PG_DUMP_BIN` is set.
|
||||
*
|
||||
* **Server/client major version:** `pg_dump` must match the server's major version or be newer (e.g.
|
||||
* PostgreSQL **15** server needs `pg_dump` **15+**, not 14). Postgres.app 14 ships `pg_dump` 14 —
|
||||
* upgrade Postgres.app, install `postgresql@15` / `libpq` from Homebrew, or set `PG_DUMP_BIN`.
|
||||
*
|
||||
* Env:
|
||||
* - `DATABASE_URL` — connection string (default source).
|
||||
* - `PGDUMP_URL` — overrides `DATABASE_URL` (use Supabase **direct** Postgres `5432` if pooler rejects dump).
|
||||
* - `PG_DUMP_BIN` — path to `pg_dump` executable (defaults to `pg_dump` on `PATH`).
|
||||
* - `PG_DUMP_FORMAT=plain` — SQL text `.sql`; default is custom `-Fc` `.dump` (`pg_restore` compatible).
|
||||
*/
|
||||
|
||||
import 'dotenv/config';
|
||||
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdirSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const connectionUrl = process.env.PGDUMP_URL ?? process.env.DATABASE_URL;
|
||||
|
||||
if (!connectionUrl) {
|
||||
console.error('Set DATABASE_URL (or PGDUMP_URL) to run a backup.');
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
const usePlainSql = process.env.PG_DUMP_FORMAT === 'plain';
|
||||
const verbose = process.env.PG_DUMP_VERBOSE !== '0';
|
||||
const pgDumpExecutable = process.env.PG_DUMP_BIN ?? 'pg_dump';
|
||||
|
||||
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
const scriptDir = path.dirname(fileURLToPath(import.meta.url));
|
||||
const backupDir = path.resolve(scriptDir, '../../backups');
|
||||
|
||||
function versionMismatchHint() {
|
||||
return [
|
||||
'',
|
||||
'If the error mentions server/client version mismatch, use a newer pg_dump (major >= server), e.g.:',
|
||||
' brew install postgresql@15',
|
||||
` PG_DUMP_BIN=$(brew --prefix postgresql@15)/bin/pg_dump pnpm --filter @cio/db db:dump-backup`,
|
||||
'(Or upgrade Postgres.app to a bundle that ships PostgreSQL 15+ client tools.)',
|
||||
''
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function main() {
|
||||
if (!connectionUrl) {
|
||||
return;
|
||||
}
|
||||
|
||||
mkdirSync(backupDir, { recursive: true });
|
||||
|
||||
const outfile = path.join(backupDir, usePlainSql ? `full-${stamp}.sql` : `full-${stamp}.dump`);
|
||||
|
||||
const args = usePlainSql
|
||||
? [...(verbose ? ['-v'] : []), '--dbname', connectionUrl, '-F', 'p', '-f', outfile]
|
||||
: [...(verbose ? ['-v'] : []), '--dbname', connectionUrl, '-F', 'c', '-Z', '9', '--file', outfile];
|
||||
|
||||
console.log('Using:', pgDumpExecutable);
|
||||
console.log('pg_dump →', outfile);
|
||||
|
||||
const result = spawnSync(pgDumpExecutable, args, { stdio: 'inherit' });
|
||||
|
||||
if (result.error) {
|
||||
console.error('pg_dump spawn failed:', result.error.message);
|
||||
console.error(versionMismatchHint());
|
||||
process.exitCode = 1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (result.status !== 0) {
|
||||
process.exitCode = result.status ?? 1;
|
||||
console.error(versionMismatchHint());
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('Backup finished:', outfile);
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,431 @@
|
||||
/**
|
||||
* Pushes the **`public` schema DDL** from a `pg_dump` backup into a fresh Postgres database
|
||||
* (e.g. [PlanetScale Postgres](https://planetscale.com/docs/postgres/imports/postgres-migrate-dumprestore)).
|
||||
* Does **not** load table data (`COPY` / `--data-only` rows).
|
||||
*
|
||||
* **Custom format** (`.dump` / `-Fc`, default from `db:dump-backup`): `pg_restore --schema-only -n public`,
|
||||
* with `--no-publications` / `--no-subscriptions` (Supabase `supabase_realtime` is not on PlanetScale).
|
||||
*
|
||||
* **Plain SQL** (`.sql`): runs the file with `psql` when it looks like a schema-only dump (no `COPY public.`
|
||||
* data blocks). Full plain dumps with data are rejected — re-dump with `-Fc` or `pg_dump --schema-only`.
|
||||
*
|
||||
* Before push, creates `authenticated` / `anon` roles and a minimal `extensions` schema
|
||||
* (`moddatetime`, `gen_random_uuid`) so Supabase-sourced dumps restore on PlanetScale.
|
||||
*
|
||||
* @example
|
||||
* DATABASE_URL="postgres://[email protected]/..." \
|
||||
* pnpm --filter @cio/db db:push-dump-to-new-db -- packages/db/backups/full-….dump
|
||||
*
|
||||
* @example
|
||||
* pnpm --filter @cio/db db:push-dump-to-new-db -- --dry-run path/to/backup.dump
|
||||
*
|
||||
* Env:
|
||||
* - `DATABASE_URL` — target DB (required unless `--dry-run`).
|
||||
* - `PG_RESTORE_BIN` — default `pg_restore`.
|
||||
* - `PSQL_BIN` — default `psql`.
|
||||
* - `PG_RESTORE_JOBS` — parallel workers for custom-format restore (e.g. `4`).
|
||||
* - `PUSH_DUMP_SSLMODE` — override `sslmode` for `pg_restore` / `psql` only (e.g. `require`).
|
||||
* - `PGSSLROOTCERT` — path to CA bundle for libpq (alternative to `sslrootcert=system` in the URL).
|
||||
*
|
||||
* PlanetScale URLs often use `sslmode=verify-full`. Node `postgres` handles that; libpq often does not
|
||||
* unless `PGSSLROOTCERT` points at a CA file. This script downgrades to `sslmode=require` for
|
||||
* `pg_restore` / `psql` when verify modes are set without a cert path (encrypted, no CA verify).
|
||||
*/
|
||||
|
||||
import 'dotenv/config';
|
||||
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import postgres from 'postgres';
|
||||
|
||||
const scriptDir = path.dirname(fileURLToPath(import.meta.url));
|
||||
const backupDir = path.resolve(scriptDir, '../../backups');
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const dryRun = args.includes('--dry-run');
|
||||
const force = args.includes('--force');
|
||||
const cleanFirst = args.includes('--clean');
|
||||
const skipRoles = args.includes('--skip-roles');
|
||||
const skipSupabaseExtensions = args.includes('--skip-supabase-extensions');
|
||||
const sslRequireOnly = args.includes('--ssl-require');
|
||||
const jobsRaw = args.find((argument) => argument.startsWith('--jobs='));
|
||||
const parallelJobs = jobsRaw ? Number(jobsRaw.slice('--jobs='.length)) : Number(process.env.PG_RESTORE_JOBS ?? 0);
|
||||
|
||||
const dumpPathRaw = args.find((argument) => !argument.startsWith('-'));
|
||||
|
||||
const databaseUrl = process.env.DATABASE_URL;
|
||||
const pgRestoreExecutable = process.env.PG_RESTORE_BIN ?? 'pg_restore';
|
||||
const psqlExecutable = process.env.PSQL_BIN ?? 'psql';
|
||||
|
||||
async function main() {
|
||||
if (!dumpPathRaw) {
|
||||
printUsage();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (!dryRun && !databaseUrl) {
|
||||
console.error('Set DATABASE_URL (or pass --dry-run to only generate a SQL preview).');
|
||||
printUsage();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const resolvedDump = path.isAbsolute(dumpPathRaw) ? dumpPathRaw : path.resolve(process.cwd(), dumpPathRaw);
|
||||
|
||||
let dumpKind: DumpKind;
|
||||
try {
|
||||
dumpKind = detectDumpKind(resolvedDump);
|
||||
} catch (error) {
|
||||
console.error(error instanceof Error ? error.message : error);
|
||||
process.exitCode = 1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('Dump:', resolvedDump);
|
||||
console.log('Format:', dumpKind === 'custom' ? 'custom (pg_restore)' : 'plain SQL (psql)');
|
||||
|
||||
try {
|
||||
if (dryRun) {
|
||||
mkdirSync(backupDir, { recursive: true });
|
||||
const previewFile = path.join(backupDir, 'push-dump-to-new-db-preview.sql');
|
||||
|
||||
if (dumpKind === 'custom') {
|
||||
extractCustomFormatSchemaPreview(resolvedDump, previewFile);
|
||||
} else {
|
||||
writeFileSync(previewFile, readFileSync(resolvedDump, 'utf8'));
|
||||
}
|
||||
|
||||
console.log('Dry run: wrote preview SQL to', previewFile);
|
||||
console.log('Apply manually with psql, or re-run without --dry-run and DATABASE_URL set.');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
await ensureTargetReady(databaseUrl!, force);
|
||||
|
||||
if (!skipRoles) {
|
||||
await ensureRlsRoles(databaseUrl!);
|
||||
}
|
||||
|
||||
if (!skipSupabaseExtensions) {
|
||||
await ensureSupabaseExtensionCompat(databaseUrl!);
|
||||
}
|
||||
|
||||
const pgCliDatabaseUrl = connectionStringForPgCli(databaseUrl!, sslRequireOnly);
|
||||
|
||||
if (dumpKind === 'custom') {
|
||||
pushCustomFormatSchema(resolvedDump, pgCliDatabaseUrl);
|
||||
} else {
|
||||
pushPlainSchemaSql(resolvedDump, pgCliDatabaseUrl);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Push dump to new DB failed:', error);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
type DumpKind = 'custom' | 'plain';
|
||||
|
||||
function printUsage() {
|
||||
console.error(`Usage:
|
||||
DATABASE_URL="postgres://..." pnpm --filter @cio/db db:push-dump-to-new-db -- <dump.dump|.sql>
|
||||
|
||||
Flags:
|
||||
--dry-run Write SQL preview to packages/db/backups/push-dump-to-new-db-preview.sql
|
||||
--force Push even when public already has tables
|
||||
--clean Drop existing public objects before push (pg_restore --clean --if-exists)
|
||||
--skip-roles Do not create authenticated / anon roles before push
|
||||
--skip-supabase-extensions Skip creating extensions.moddatetime / gen_random_uuid stubs
|
||||
--jobs=N Parallel pg_restore workers (custom .dump only)
|
||||
--ssl-require Force sslmode=require for pg_restore/psql (same as the default when no CA file)`);
|
||||
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
function detectDumpKind(dumpFilePath: string): DumpKind {
|
||||
const header = readFileSync(dumpFilePath).subarray(0, 5).toString();
|
||||
|
||||
if (header === 'PGDMP') {
|
||||
return 'custom';
|
||||
}
|
||||
|
||||
const plainText = readFileSync(dumpFilePath, 'utf8');
|
||||
|
||||
if (/\bCOPY\s+public\./i.test(plainText)) {
|
||||
throw new Error(
|
||||
'Plain SQL dump contains public COPY data blocks. Use a custom-format .dump from db:dump-backup, or pg_dump --schema-only -n public.'
|
||||
);
|
||||
}
|
||||
|
||||
return 'plain';
|
||||
}
|
||||
|
||||
async function ensureTargetReady(connectionString: string, allowExistingTables: boolean) {
|
||||
const sql = postgres(connectionString, { max: 1 });
|
||||
|
||||
try {
|
||||
const existingTables = await sql<{ tablename: string }[]>`
|
||||
SELECT tablename
|
||||
FROM pg_tables
|
||||
WHERE schemaname = 'public'
|
||||
ORDER BY tablename
|
||||
`;
|
||||
|
||||
if (existingTables.length === 0) {
|
||||
console.log('Target public schema is empty.');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (!allowExistingTables) {
|
||||
throw new Error(
|
||||
`Target already has ${existingTables.length} table(s) in public (e.g. ${existingTables[0]?.tablename}). ` +
|
||||
'Use --force to push anyway, or provision a fresh database branch.'
|
||||
);
|
||||
}
|
||||
|
||||
console.warn(
|
||||
`--force: continuing despite ${existingTables.length} existing public table(s).`,
|
||||
cleanFirst ? ' (--clean will drop objects first)' : ''
|
||||
);
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Supabase dumps reference `extensions.moddatetime` triggers and sometimes `extensions.gen_random_uuid`.
|
||||
* PlanetScale has no `extensions` schema; create minimal compatible stubs before pg_restore.
|
||||
*/
|
||||
async function ensureSupabaseExtensionCompat(connectionString: string) {
|
||||
const sql = postgres(connectionString, { max: 1 });
|
||||
|
||||
try {
|
||||
await sql.unsafe(`
|
||||
CREATE SCHEMA IF NOT EXISTS extensions;
|
||||
|
||||
CREATE OR REPLACE FUNCTION extensions.gen_random_uuid()
|
||||
RETURNS uuid
|
||||
LANGUAGE sql
|
||||
PARALLEL SAFE
|
||||
AS $$ SELECT gen_random_uuid(); $$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION extensions.moddatetime()
|
||||
RETURNS trigger
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = NOW();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION extensions.moddatetime(column_name text)
|
||||
RETURNS trigger
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
BEGIN
|
||||
IF column_name = 'updated_at' THEN
|
||||
NEW.updated_at = NOW();
|
||||
ELSE
|
||||
NEW := jsonb_populate_record(NEW, jsonb_build_object(column_name, NOW()));
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
`);
|
||||
|
||||
console.log('Ensured extensions schema (moddatetime, gen_random_uuid) for Supabase-compatible dumps');
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
}
|
||||
|
||||
async function ensureRlsRoles(connectionString: string) {
|
||||
const sql = postgres(connectionString, { max: 1 });
|
||||
|
||||
try {
|
||||
const authenticatedExists = await sql`SELECT 1 FROM pg_roles WHERE rolname = 'authenticated'`;
|
||||
if (authenticatedExists.length === 0) {
|
||||
await sql`CREATE ROLE authenticated`;
|
||||
console.log('Created role: authenticated');
|
||||
} else {
|
||||
console.log('Role already exists: authenticated');
|
||||
}
|
||||
|
||||
const anonExists = await sql`SELECT 1 FROM pg_roles WHERE rolname = 'anon'`;
|
||||
if (anonExists.length === 0) {
|
||||
await sql`CREATE ROLE anon`;
|
||||
console.log('Created role: anon');
|
||||
} else {
|
||||
console.log('Role already exists: anon');
|
||||
}
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
}
|
||||
|
||||
function pushCustomFormatSchema(dumpFilePath: string, connectionString: string) {
|
||||
const restoreArgs = [
|
||||
'--dbname',
|
||||
connectionString,
|
||||
'--schema-only',
|
||||
'--no-owner',
|
||||
'--no-privileges',
|
||||
'--no-publications',
|
||||
'--no-subscriptions',
|
||||
'--exit-on-error',
|
||||
'-n',
|
||||
'public',
|
||||
...(cleanFirst ? ['--clean', '--if-exists'] : []),
|
||||
...(parallelJobs > 0 ? ['--jobs', String(parallelJobs)] : []),
|
||||
'--verbose',
|
||||
dumpFilePath
|
||||
];
|
||||
|
||||
console.log('Running:', pgRestoreExecutable, restoreArgs.filter((argument) => !argument.includes('://')).join(' '));
|
||||
|
||||
const result = spawnSync(pgRestoreExecutable, restoreArgs, { stdio: 'inherit' });
|
||||
|
||||
if (result.error) {
|
||||
throw new Error(`pg_restore spawn failed: ${result.error.message}`);
|
||||
}
|
||||
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`pg_restore exited with status ${result.status ?? 'unknown'}`);
|
||||
}
|
||||
|
||||
console.log('Public schema push finished.');
|
||||
console.log(
|
||||
'Next: run pnpm --filter @cio/db db:setup --skip-schema-sync (roles + reference seed), then data restores as needed.'
|
||||
);
|
||||
}
|
||||
|
||||
function pushPlainSchemaSql(dumpFilePath: string, connectionString: string) {
|
||||
console.log('Running:', psqlExecutable, '<DATABASE_URL> -v ON_ERROR_STOP=1 -f', dumpFilePath);
|
||||
|
||||
const result = spawnSync(psqlExecutable, [connectionString, '-v', 'ON_ERROR_STOP=1', '-f', dumpFilePath], {
|
||||
stdio: 'inherit'
|
||||
});
|
||||
|
||||
if (result.error) {
|
||||
throw new Error(`psql spawn failed: ${result.error.message}`);
|
||||
}
|
||||
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`psql exited with status ${result.status ?? 'unknown'}`);
|
||||
}
|
||||
|
||||
console.log('Public schema SQL applied.');
|
||||
}
|
||||
|
||||
function extractCustomFormatSchemaPreview(dumpFilePath: string, previewFilePath: string) {
|
||||
const extractArgs = [
|
||||
'--schema-only',
|
||||
'--no-owner',
|
||||
'--no-privileges',
|
||||
'-n',
|
||||
'public',
|
||||
'-f',
|
||||
previewFilePath,
|
||||
dumpFilePath
|
||||
];
|
||||
|
||||
const result = spawnSync(pgRestoreExecutable, extractArgs, { encoding: 'utf8' });
|
||||
|
||||
if (result.error) {
|
||||
throw new Error(`pg_restore preview spawn failed: ${result.error.message}`);
|
||||
}
|
||||
|
||||
if (result.status !== 0) {
|
||||
throw new Error(
|
||||
`pg_restore preview failed: ${(result.stderr ?? '').trim() || `exit ${result.status}`}. ` +
|
||||
'Ensure pg_restore major version matches the dump.'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* libpq (`pg_restore`, `psql`) does not use Node's TLS stack. `verify-full` without a CA file makes
|
||||
* libpq look for `~/.postgresql/root.crt`. Older libpq also treats `sslrootcert=system` as a path.
|
||||
* Use `sslmode=require` unless the URL or `PGSSLROOTCERT` supplies a real certificate file.
|
||||
*/
|
||||
function connectionStringForPgCli(rawUrl: string, useSslRequireOnly: boolean): string {
|
||||
const url = new URL(rawUrl.replace(/^postgres:/, 'postgresql:'));
|
||||
|
||||
const envSslMode = process.env.PUSH_DUMP_SSLMODE?.trim();
|
||||
if (envSslMode) {
|
||||
applyPgCliSslMode(url, envSslMode);
|
||||
console.log(`pg_restore/psql: using PUSH_DUMP_SSLMODE=${envSslMode}`);
|
||||
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
if (useSslRequireOnly) {
|
||||
applyPgCliSslMode(url, 'require');
|
||||
console.log('pg_restore/psql: using sslmode=require (--ssl-require)');
|
||||
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
const sslmode = url.searchParams.get('sslmode') ?? '';
|
||||
const rootCertPath = resolvePgCliRootCertPath(url);
|
||||
|
||||
if (rootCertPath) {
|
||||
url.searchParams.set('sslrootcert', rootCertPath);
|
||||
if (!sslmode) {
|
||||
url.searchParams.set('sslmode', 'verify-full');
|
||||
}
|
||||
|
||||
console.log(`pg_restore/psql: using sslrootcert=${rootCertPath}`);
|
||||
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
const needsEncryptedConnection =
|
||||
sslmode === 'verify-full' ||
|
||||
sslmode === 'verify-ca' ||
|
||||
sslmode === 'require' ||
|
||||
url.hostname.endsWith('.psdb.cloud');
|
||||
|
||||
if (needsEncryptedConnection && sslmode !== 'require') {
|
||||
applyPgCliSslMode(url, 'require');
|
||||
console.log(
|
||||
'pg_restore/psql: using sslmode=require (libpq has no CA file; connection is still TLS-encrypted). ' +
|
||||
'Set PGSSLROOTCERT or PUSH_DUMP_SSLMODE=verify-full to verify the server certificate.'
|
||||
);
|
||||
}
|
||||
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
function applyPgCliSslMode(url: URL, sslmode: string) {
|
||||
url.searchParams.set('sslmode', sslmode);
|
||||
|
||||
if (sslmode === 'require' || sslmode === 'prefer' || sslmode === 'disable') {
|
||||
url.searchParams.delete('sslrootcert');
|
||||
}
|
||||
}
|
||||
|
||||
function resolvePgCliRootCertPath(url: URL): string | null {
|
||||
const envPath = process.env.PGSSLROOTCERT?.trim();
|
||||
if (envPath) {
|
||||
return envPath;
|
||||
}
|
||||
|
||||
const paramPath = url.searchParams.get('sslrootcert')?.trim();
|
||||
if (!paramPath || paramPath === 'system') {
|
||||
return null;
|
||||
}
|
||||
|
||||
return paramPath;
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1,235 @@
|
||||
/**
|
||||
* Copy `public.lesson_section` **data** from a `pg_dump` backup into **live** `public.course_section`
|
||||
* on `DATABASE_URL` (UPSERT on `id`).
|
||||
*
|
||||
* **Custom format** (`.dump` / `-Fc`): uses `pg_restore --data-only -t lesson_section` (needs version
|
||||
* compatible with the dump, same as `pg_dump`).
|
||||
*
|
||||
* **Plain SQL** (`-Fp` `.sql`): scans for a `COPY ... lesson_section ... FROM stdin` block.
|
||||
*
|
||||
* Usage:
|
||||
* pnpm --filter @cio/db db:restore-lesson-section-from-dump -- path/to/backup.dump
|
||||
* pnpm exec tsx src/scripts/restore-lesson-section-from-dump.ts --dry-run path/to/backup.sql
|
||||
*
|
||||
* Workflow: If `pnpm db migrate` halted on FK errors because `course_section` had no rows yet, restore
|
||||
* sections from your dump with this script, then run `pnpm db migrate` again—the migration uses conditional
|
||||
* `DO` blocks that add missing FKs when referenced ids exist.
|
||||
*
|
||||
* Env:
|
||||
* - `DATABASE_URL` — target DB (required unless `--dry-run`).
|
||||
* - `PG_RESTORE_BIN` — `pg_restore` binary (default `pg_restore`).
|
||||
* - `PSQL_BIN` — `psql` binary (default `psql`).
|
||||
*/
|
||||
|
||||
import 'dotenv/config';
|
||||
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const scriptDir = path.dirname(fileURLToPath(import.meta.url));
|
||||
const backupDir = path.resolve(scriptDir, '../../backups');
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const dryRun = args.includes('--dry-run');
|
||||
const dumpPathRaw = args.find((argument) => !argument.startsWith('-'));
|
||||
|
||||
const databaseUrl = process.env.DATABASE_URL;
|
||||
const pgRestoreExecutable = process.env.PG_RESTORE_BIN ?? 'pg_restore';
|
||||
const psqlExecutable = process.env.PSQL_BIN ?? 'psql';
|
||||
|
||||
const prelude = `
|
||||
BEGIN;
|
||||
CREATE TEMP TABLE _lesson_section_import (
|
||||
id uuid NOT NULL PRIMARY KEY,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz,
|
||||
title varchar,
|
||||
"order" bigint,
|
||||
course_id uuid
|
||||
);
|
||||
|
||||
`;
|
||||
|
||||
const mergeSql = `
|
||||
INSERT INTO public.course_section (id, created_at, updated_at, title, "order", course_id)
|
||||
SELECT id, created_at, updated_at, title, "order", course_id
|
||||
FROM _lesson_section_import
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
created_at = EXCLUDED.created_at,
|
||||
updated_at = EXCLUDED.updated_at,
|
||||
title = EXCLUDED.title,
|
||||
"order" = EXCLUDED."order",
|
||||
course_id = EXCLUDED.course_id;
|
||||
|
||||
COMMIT;
|
||||
`;
|
||||
|
||||
function renameCopyTargets(body: string) {
|
||||
let out = body;
|
||||
out = out.replace(/\bCOPY\s+public\.lesson_section\b/i, 'COPY _lesson_section_import');
|
||||
out = out.replace(/\bCOPY\s+lesson_section\b/i, 'COPY _lesson_section_import');
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Plain-format dump: one COPY ... lesson_section ... FROM stdin; … \\. block */
|
||||
function slicePlainCopyLessonSection(fileText: string) {
|
||||
const copyPublic = /\bCOPY\s+public\.lesson_section\b/i.exec(fileText);
|
||||
const copyBare = /\bCOPY\s+lesson_section\b/i.exec(fileText);
|
||||
|
||||
const matchIndex = copyPublic?.index ?? copyBare?.index;
|
||||
if (matchIndex === undefined) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const fromStdinIdx = fileText.indexOf('FROM stdin;', matchIndex);
|
||||
|
||||
if (fromStdinIdx === -1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let lineEnd = fileText.indexOf('\n', fromStdinIdx + 'FROM stdin;'.length);
|
||||
|
||||
if (lineEnd === -1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
/** First line after DATA is first row; terminator is a line `\`. */
|
||||
const terminator = /\n\\\.\s*(?:\n|\r?\n|$)/;
|
||||
|
||||
const dataRegion = fileText.slice(lineEnd);
|
||||
const terminatorMatch = terminator.exec(dataRegion);
|
||||
|
||||
if (!terminatorMatch) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const terminatorEndAbs = lineEnd + terminatorMatch.index + terminatorMatch[0].length;
|
||||
|
||||
return fileText.slice(matchIndex, terminatorEndAbs);
|
||||
}
|
||||
|
||||
function pgRestoreLessonSectionExtract(dumpFilePath: string) {
|
||||
const tempDir = mkdtempSync(path.join(tmpdir(), 'lesson-section-restore-'));
|
||||
const extractedPath = path.join(tempDir, 'lesson_section.sql');
|
||||
|
||||
try {
|
||||
const flags = spawnSync(
|
||||
pgRestoreExecutable,
|
||||
[
|
||||
'--data-only',
|
||||
'--no-owner',
|
||||
'--no-privileges',
|
||||
'-n',
|
||||
'public',
|
||||
'-t',
|
||||
'lesson_section',
|
||||
'-f',
|
||||
extractedPath,
|
||||
dumpFilePath
|
||||
],
|
||||
{ encoding: 'utf8' }
|
||||
);
|
||||
|
||||
if (flags.status !== 0) {
|
||||
console.error(pgRestoreExecutable, '--data-only extraction failed:', (flags.stderr ?? '').trim() || flags.status);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
return renameCopyTargets(readFileSync(extractedPath, 'utf8'));
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function printUsage() {
|
||||
console.error(`Usage:
|
||||
DATABASE_URL="postgres://..." pnpm exec tsx src/scripts/restore-lesson-section-from-dump.ts <dump.dump|.sql>
|
||||
|
||||
Flags:
|
||||
--dry-run Write merged SQL preview to packages/db/backups/restore-lesson-section-preview.sql`);
|
||||
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
function main() {
|
||||
if (!dumpPathRaw) {
|
||||
console.error('Missing dump file path.');
|
||||
printUsage();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (!dryRun && !databaseUrl) {
|
||||
console.error('Set DATABASE_URL (or pass --dry-run to only generate SQL preview).');
|
||||
|
||||
printUsage();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const resolvedDump = path.isAbsolute(dumpPathRaw) ? dumpPathRaw : path.resolve(process.cwd(), dumpPathRaw);
|
||||
|
||||
let lessonSectionChunk: string | null;
|
||||
|
||||
if (resolvedDump.endsWith('.sql')) {
|
||||
const text = readFileSync(resolvedDump, 'utf8');
|
||||
lessonSectionChunk = slicePlainCopyLessonSection(text);
|
||||
if (lessonSectionChunk) {
|
||||
lessonSectionChunk = renameCopyTargets(lessonSectionChunk);
|
||||
}
|
||||
} else {
|
||||
lessonSectionChunk = pgRestoreLessonSectionExtract(resolvedDump);
|
||||
}
|
||||
|
||||
if (!lessonSectionChunk) {
|
||||
console.error(
|
||||
'Could not extract lesson_section data. Check:',
|
||||
`\n - Table exists in backup as public.lesson_section`,
|
||||
`\n - For custom dumps: pg_restore -l "${resolvedDump}" | grep lesson_section`,
|
||||
`\n - pg_restore major version matches the dump`
|
||||
);
|
||||
process.exitCode = 1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const fullScript = prelude + lessonSectionChunk + mergeSql;
|
||||
|
||||
mkdirSync(backupDir, { recursive: true });
|
||||
|
||||
const previewFile = path.join(backupDir, 'restore-lesson-section-preview.sql');
|
||||
writeFileSync(previewFile, fullScript, 'utf8');
|
||||
console.log('Wrote SQL preview:', previewFile);
|
||||
|
||||
if (dryRun) {
|
||||
console.log('Dry run: skipping psql execution.');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const invoke = spawnSync(psqlExecutable, [databaseUrl!, '-v', 'ON_ERROR_STOP=1', '-f', previewFile], {
|
||||
stdio: 'inherit'
|
||||
});
|
||||
|
||||
if (invoke.error) {
|
||||
console.error('psql spawn failed:', invoke.error.message);
|
||||
process.exitCode = 1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (invoke.status !== 0) {
|
||||
process.exitCode = invoke.status ?? 1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('Done: lesson_section rows from dump were UPSERT-ed into course_section via', previewFile);
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,272 @@
|
||||
/**
|
||||
* Insert `public.organizationmember` **rows from a dump** into the DB on `DATABASE_URL` **only when
|
||||
* missing** relative to existing data:
|
||||
* - skip if **`id`** already exists;
|
||||
* - skip if **`(organization_id, profile_id)`** already exists (**`profile_id IS NOT NULL`**, aligned with
|
||||
* partial unique `organizationmember_org_profile_unique`);
|
||||
* - skip if **`(organization_id, email)`** already exists (**`email IS NOT NULL`**, aligned with
|
||||
* `organizationmember_org_email_unique`).
|
||||
*
|
||||
* **Custom format** (`.dump` / `-Fc`): `pg_restore --data-only -t organizationmember`.
|
||||
* **Plain SQL** (`-Fp` `.sql`): finds `COPY ... organizationmember ... FROM stdin` block.
|
||||
*
|
||||
* Usage:
|
||||
* DATABASE_URL="postgres://..." pnpm exec tsx src/scripts/restore-organizationmember-from-dump.ts path/to/backup.dump
|
||||
* pnpm exec tsx src/scripts/restore-organizationmember-from-dump.ts --dry-run path/to/backup.sql
|
||||
*
|
||||
* Requires existing `organization`, `profile` (when `profile_id` set), and `role` rows for FKs.
|
||||
*
|
||||
* Env:
|
||||
* - `DATABASE_URL` — target DB (required unless `--dry-run`).
|
||||
* - `PG_RESTORE_BIN` — default `pg_restore`.
|
||||
* - `PSQL_BIN` — default `psql`.
|
||||
*/
|
||||
|
||||
import 'dotenv/config';
|
||||
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const scriptDir = path.dirname(fileURLToPath(import.meta.url));
|
||||
const backupDir = path.resolve(scriptDir, '../../backups');
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const dryRun = args.includes('--dry-run');
|
||||
const dumpPathRaw = args.find((argument) => !argument.startsWith('-'));
|
||||
|
||||
const databaseUrl = process.env.DATABASE_URL;
|
||||
const pgRestoreExecutable = process.env.PG_RESTORE_BIN ?? 'pg_restore';
|
||||
const psqlExecutable = process.env.PSQL_BIN ?? 'psql';
|
||||
|
||||
/** Column order matches `organizationmember` in schema / typical pg_dump. */
|
||||
const prelude = `
|
||||
BEGIN;
|
||||
|
||||
CREATE TEMP TABLE _organizationmember_import (
|
||||
id bigint NOT NULL PRIMARY KEY,
|
||||
organization_id uuid NOT NULL,
|
||||
role_id bigint NOT NULL,
|
||||
profile_id uuid,
|
||||
email text,
|
||||
verified boolean,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
`;
|
||||
|
||||
const mergeSql = `
|
||||
INSERT INTO public.organizationmember (
|
||||
id,
|
||||
organization_id,
|
||||
role_id,
|
||||
profile_id,
|
||||
email,
|
||||
verified,
|
||||
created_at
|
||||
)
|
||||
SELECT
|
||||
i.id,
|
||||
i.organization_id,
|
||||
i.role_id,
|
||||
i.profile_id,
|
||||
i.email,
|
||||
COALESCE(i.verified, false),
|
||||
i.created_at
|
||||
FROM _organizationmember_import i
|
||||
WHERE NOT EXISTS (SELECT 1 FROM public.organizationmember om WHERE om.id = i.id)
|
||||
AND (
|
||||
i.profile_id IS NULL
|
||||
OR NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM public.organizationmember om
|
||||
WHERE om.organization_id = i.organization_id
|
||||
AND om.profile_id IS NOT NULL
|
||||
AND om.profile_id = i.profile_id
|
||||
)
|
||||
)
|
||||
AND (
|
||||
i.email IS NULL
|
||||
OR NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM public.organizationmember om
|
||||
WHERE om.organization_id = i.organization_id
|
||||
AND om.email IS NOT NULL
|
||||
AND om.email = i.email
|
||||
)
|
||||
);
|
||||
|
||||
SELECT setval(
|
||||
pg_get_serial_sequence('public.organizationmember', 'id'),
|
||||
COALESCE((SELECT MAX(id)::bigint FROM public.organizationmember), 1),
|
||||
true
|
||||
);
|
||||
|
||||
COMMIT;
|
||||
`;
|
||||
|
||||
function renameCopyTargets(body: string) {
|
||||
let out = body;
|
||||
out = out.replace(/\bCOPY\s+public\.organizationmember\b/i, 'COPY _organizationmember_import');
|
||||
out = out.replace(/\bCOPY\s+organizationmember\b/i, 'COPY _organizationmember_import');
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
function slicePlainCopyOrganizationMember(fileText: string) {
|
||||
const copyPublic = /\bCOPY\s+public\.organizationmember\b/i.exec(fileText);
|
||||
const copyBare = /\bCOPY\s+organizationmember\b/i.exec(fileText);
|
||||
|
||||
const matchIndex = copyPublic?.index ?? copyBare?.index;
|
||||
if (matchIndex === undefined) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const fromStdinIdx = fileText.indexOf('FROM stdin;', matchIndex);
|
||||
|
||||
if (fromStdinIdx === -1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let lineEnd = fileText.indexOf('\n', fromStdinIdx + 'FROM stdin;'.length);
|
||||
|
||||
if (lineEnd === -1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const terminator = /\n\\\.\s*(?:\n|\r?\n|$)/;
|
||||
|
||||
const dataRegion = fileText.slice(lineEnd);
|
||||
const terminatorMatch = terminator.exec(dataRegion);
|
||||
|
||||
if (!terminatorMatch) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const terminatorEndAbs = lineEnd + terminatorMatch.index + terminatorMatch[0].length;
|
||||
|
||||
return fileText.slice(matchIndex, terminatorEndAbs);
|
||||
}
|
||||
|
||||
function pgRestoreOrganizationMemberExtract(dumpFilePath: string) {
|
||||
const tempDir = mkdtempSync(path.join(tmpdir(), 'organizationmember-restore-'));
|
||||
const extractedPath = path.join(tempDir, 'organizationmember.sql');
|
||||
|
||||
try {
|
||||
const flags = spawnSync(
|
||||
pgRestoreExecutable,
|
||||
[
|
||||
'--data-only',
|
||||
'--no-owner',
|
||||
'--no-privileges',
|
||||
'-n',
|
||||
'public',
|
||||
'-t',
|
||||
'organizationmember',
|
||||
'-f',
|
||||
extractedPath,
|
||||
dumpFilePath
|
||||
],
|
||||
{ encoding: 'utf8' }
|
||||
);
|
||||
|
||||
if (flags.status !== 0) {
|
||||
console.error(pgRestoreExecutable, '--data-only extraction failed:', (flags.stderr ?? '').trim() || flags.status);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
return renameCopyTargets(readFileSync(extractedPath, 'utf8'));
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function printUsage() {
|
||||
console.error(`Usage:
|
||||
DATABASE_URL="postgres://..." pnpm exec tsx src/scripts/restore-organizationmember-from-dump.ts <dump.dump|.sql>
|
||||
|
||||
Flags:
|
||||
--dry-run Write merged SQL preview to packages/db/backups/restore-organizationmember-preview.sql`);
|
||||
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
function main() {
|
||||
if (!dumpPathRaw) {
|
||||
console.error('Missing dump file path.');
|
||||
printUsage();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (!dryRun && !databaseUrl) {
|
||||
console.error('Set DATABASE_URL (or pass --dry-run to only generate SQL preview).');
|
||||
printUsage();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const resolvedDump = path.isAbsolute(dumpPathRaw) ? dumpPathRaw : path.resolve(process.cwd(), dumpPathRaw);
|
||||
|
||||
let organizationMemberChunk: string | null;
|
||||
|
||||
if (resolvedDump.endsWith('.sql')) {
|
||||
const text = readFileSync(resolvedDump, 'utf8');
|
||||
organizationMemberChunk = slicePlainCopyOrganizationMember(text);
|
||||
if (organizationMemberChunk) {
|
||||
organizationMemberChunk = renameCopyTargets(organizationMemberChunk);
|
||||
}
|
||||
} else {
|
||||
organizationMemberChunk = pgRestoreOrganizationMemberExtract(resolvedDump);
|
||||
}
|
||||
|
||||
if (!organizationMemberChunk) {
|
||||
console.error(
|
||||
'Could not extract organizationmember data. Check:',
|
||||
`\n - Table exists in backup as public.organizationmember`,
|
||||
`\n - For custom dumps: pg_restore -l "${resolvedDump}" | grep organizationmember`,
|
||||
`\n - pg_restore major version matches the dump`
|
||||
);
|
||||
process.exitCode = 1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const fullScript = prelude + organizationMemberChunk + mergeSql;
|
||||
|
||||
mkdirSync(backupDir, { recursive: true });
|
||||
|
||||
const previewFile = path.join(backupDir, 'restore-organizationmember-preview.sql');
|
||||
writeFileSync(previewFile, fullScript, 'utf8');
|
||||
console.log('Wrote SQL preview:', previewFile);
|
||||
|
||||
if (dryRun) {
|
||||
console.log('Dry run: skipping psql execution.');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const invoke = spawnSync(psqlExecutable, [databaseUrl!, '-v', 'ON_ERROR_STOP=1', '-f', previewFile], {
|
||||
stdio: 'inherit'
|
||||
});
|
||||
|
||||
if (invoke.error) {
|
||||
console.error('psql spawn failed:', invoke.error.message);
|
||||
process.exitCode = 1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (invoke.status !== 0) {
|
||||
process.exitCode = invoke.status ?? 1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('Done: organizationmember rows from dump were merged (missing-only) via', previewFile);
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -1,5 +1,13 @@
|
||||
/**
|
||||
* Commenting this script out.
|
||||
* One-off migration: copies `auth.users` + identities into Better Auth `"user"` and `account`.
|
||||
*
|
||||
* Optional reset: when `RESET_BETTER_AUTH_BEFORE_SUPABASE_MIGRATE=1`, clears Better Auth targets
|
||||
* in a transaction (see `resetBetterAuthTables`) before re-import.
|
||||
*
|
||||
* Optional backfill: when `BACKFILL_AUTH_ACCOUNTS_FROM_SUPABASE=1`, only processes Supabase users
|
||||
* that already exist in `"user"` but have no rows in `account`, and creates missing account rows
|
||||
* from `auth.identities` (does not create or update `"user"` rows). Incompatible with reset;
|
||||
* reset is skipped if both flags are set (with a warning).
|
||||
*/
|
||||
|
||||
import 'dotenv/config';
|
||||
@@ -9,6 +17,87 @@ import { admin, anonymous } from 'better-auth/plugins';
|
||||
import { Pool } from 'pg';
|
||||
import { betterAuth } from 'better-auth';
|
||||
|
||||
const RESET_FLAG = process.env.RESET_BETTER_AUTH_BEFORE_SUPABASE_MIGRATE === '1';
|
||||
const BACKFILL_ACCOUNTS_ONLY = process.env.BACKFILL_AUTH_ACCOUNTS_FROM_SUPABASE === '1';
|
||||
|
||||
console.log('RESET_FLAG', RESET_FLAG);
|
||||
console.log('BACKFILL_AUTH_ACCOUNTS_FROM_SUPABASE', BACKFILL_ACCOUNTS_ONLY);
|
||||
|
||||
if (RESET_FLAG && BACKFILL_ACCOUNTS_ONLY) {
|
||||
console.warn(
|
||||
'Both RESET_BETTER_AUTH_BEFORE_SUPABASE_MIGRATE and BACKFILL_AUTH_ACCOUNTS_FROM_SUPABASE are set; skipping reset and running account backfill only.'
|
||||
);
|
||||
}
|
||||
|
||||
async function resetBetterAuthTables(pool: Pool) {
|
||||
const client = await pool.connect();
|
||||
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
await client.query('TRUNCATE TABLE analytics_login_events');
|
||||
await client.query('ALTER TABLE profile DROP CONSTRAINT IF EXISTS profile_id_fkey');
|
||||
await client.query('TRUNCATE TABLE verification');
|
||||
await client.query('DELETE FROM "user"');
|
||||
await client.query('COMMIT');
|
||||
console.log(
|
||||
'Reset complete: truncated analytics_login_events and verification; dropped profile_id_fkey; deleted all \"user\" rows (account/session cascade).'
|
||||
);
|
||||
} catch (error) {
|
||||
await client.query('ROLLBACK');
|
||||
|
||||
console.error('resetBetterAuthTables error:', error);
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function restoreProfileUserFkey(pool: Pool) {
|
||||
console.log('restoreProfileUserFkey');
|
||||
const client = await pool.connect();
|
||||
|
||||
try {
|
||||
const orphanResult = await client.query(`
|
||||
SELECT COUNT(*)::bigint AS cnt
|
||||
FROM profile p
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM "user" u
|
||||
WHERE u.id = p.id
|
||||
)
|
||||
`);
|
||||
const orphanCount = Number(orphanResult.rows[0]?.cnt ?? 0);
|
||||
|
||||
if (orphanCount > 0) {
|
||||
throw new Error(
|
||||
`Cannot re-add profile_id_fkey: ${orphanCount} profile row(s) have no matching \"user\". Remove orphan profiles or create missing users before re-running.`
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
await client.query(`
|
||||
ALTER TABLE profile
|
||||
ADD CONSTRAINT profile_id_fkey FOREIGN KEY (id) REFERENCES "user"(id)
|
||||
`);
|
||||
console.log('Restored constraint profile_id_fkey on profile(id) -> \"user\"(id).');
|
||||
} catch (addConstraintError: unknown) {
|
||||
const pgError = addConstraintError as { code?: string; message?: string };
|
||||
|
||||
// Avoid failing if FK already exists from a retry or interrupted run
|
||||
|
||||
if (pgError.code === '42710' || pgError.message?.includes('already exists')) {
|
||||
console.warn('profile_id_fkey already present; skipping ADD CONSTRAINT.', pgError.message);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
throw addConstraintError;
|
||||
}
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
// We create a custom betterAuth here because this code works well with pg
|
||||
const auth = betterAuth({
|
||||
database: new Pool({
|
||||
@@ -25,7 +114,32 @@ const auth = betterAuth({
|
||||
clientSecret: process.env.GOOGLE_CLIENT_SECRET!
|
||||
}
|
||||
},
|
||||
plugins: [admin(), anonymous()],
|
||||
plugins: [
|
||||
admin({
|
||||
schema: {
|
||||
user: {
|
||||
fields: {
|
||||
banReason: 'ban_reason',
|
||||
banExpires: 'ban_expires'
|
||||
}
|
||||
},
|
||||
session: {
|
||||
fields: {
|
||||
impersonatedBy: 'impersonated_by'
|
||||
}
|
||||
}
|
||||
}
|
||||
}),
|
||||
anonymous({
|
||||
schema: {
|
||||
user: {
|
||||
fields: {
|
||||
isAnonymous: 'is_anonymous'
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
],
|
||||
|
||||
user: {
|
||||
modelName: 'user',
|
||||
@@ -99,24 +213,62 @@ type User = SupabaseUser & {
|
||||
updated_at: string;
|
||||
is_anonymous: boolean;
|
||||
profile_fullname: string;
|
||||
identities: {
|
||||
id: string; // auth.identities.id (UUID) - use as account.id so DB accepts it
|
||||
provider: string;
|
||||
identity_data: {
|
||||
sub: string;
|
||||
email: string;
|
||||
};
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
};
|
||||
identities:
|
||||
| {
|
||||
id: string; // auth.identities.id (UUID) - use as account.id so DB accepts it
|
||||
provider: string;
|
||||
identity_data: {
|
||||
sub: string;
|
||||
email: string;
|
||||
};
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}[]
|
||||
| string;
|
||||
};
|
||||
|
||||
function parseIdentities(value: User['identities']) {
|
||||
if (Array.isArray(value)) {
|
||||
return value;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(value as string);
|
||||
|
||||
return Array.isArray(parsed) ? parsed : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
const migrateFromSupabase = async () => {
|
||||
const ctx = await auth.$context;
|
||||
const db = ctx.options.database;
|
||||
const users = await db
|
||||
.query(
|
||||
`
|
||||
const db = ctx.options.database as Pool;
|
||||
|
||||
const effectiveReset = RESET_FLAG && !BACKFILL_ACCOUNTS_ONLY;
|
||||
|
||||
if (effectiveReset) {
|
||||
await resetBetterAuthTables(db);
|
||||
}
|
||||
|
||||
const migrationWhere = `
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM "user" existing_user
|
||||
WHERE existing_user.id = u.id
|
||||
)
|
||||
`;
|
||||
|
||||
const backfillAccountsWhere = `
|
||||
INNER JOIN "user" existing_ba_user ON existing_ba_user.id = u.id
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM account a
|
||||
WHERE a.user_id = u.id
|
||||
)
|
||||
`;
|
||||
|
||||
const usersQuery = `
|
||||
SELECT
|
||||
u.*,
|
||||
COALESCE(
|
||||
@@ -129,52 +281,58 @@ const migrateFromSupabase = async () => {
|
||||
FROM auth.users u
|
||||
LEFT JOIN auth.identities i ON u.id = i.user_id
|
||||
LEFT JOIN public.profile p ON u.id = p.id -- LEFT JOIN with public.profile
|
||||
${BACKFILL_ACCOUNTS_ONLY ? backfillAccountsWhere : migrationWhere}
|
||||
GROUP BY u.id, p.fullname
|
||||
`
|
||||
)
|
||||
.then((res: { rows: User[] }) => res.rows);
|
||||
`;
|
||||
|
||||
console.log('users length', users.length);
|
||||
const users = await db.query(usersQuery).then((res: { rows: User[] }) => res.rows);
|
||||
|
||||
console.log(
|
||||
BACKFILL_ACCOUNTS_ONLY
|
||||
? 'users needing account backfill (have "user", no account rows)'
|
||||
: 'users to migrate (not yet in "user")',
|
||||
users.length
|
||||
);
|
||||
|
||||
for (const user of users) {
|
||||
if (!user.email) {
|
||||
continue;
|
||||
}
|
||||
const userName = user.profile_fullname || user.email;
|
||||
console.log('inserting user', user.email);
|
||||
if (!BACKFILL_ACCOUNTS_ONLY) {
|
||||
const userName = user.profile_fullname || user.email;
|
||||
console.log('inserting user', user.email);
|
||||
|
||||
let shouldContinue = false;
|
||||
await ctx.adapter
|
||||
.create({
|
||||
model: 'user',
|
||||
forceAllowId: true,
|
||||
data: {
|
||||
id: user.id,
|
||||
email: user.email,
|
||||
name: userName,
|
||||
role: user.is_super_admin ? 'admin' : user.role,
|
||||
emailVerified: !!user.email_confirmed_at,
|
||||
image: user.raw_user_meta_data?.avatar_url || '',
|
||||
createdAt: new Date(user.created_at),
|
||||
updatedAt: new Date(user.updated_at)
|
||||
}
|
||||
})
|
||||
.catch((e) => {
|
||||
console.log('catch');
|
||||
shouldContinue = true;
|
||||
});
|
||||
|
||||
if (shouldContinue) {
|
||||
console.log('Continue');
|
||||
// continue;
|
||||
try {
|
||||
await ctx.adapter.create({
|
||||
model: 'user',
|
||||
forceAllowId: true,
|
||||
data: {
|
||||
id: user.id,
|
||||
email: user.email,
|
||||
name: userName,
|
||||
role: user.is_super_admin ? 'admin' : user.role,
|
||||
emailVerified: !!user.email_confirmed_at,
|
||||
image: user.raw_user_meta_data?.avatar_url || '',
|
||||
createdAt: new Date(user.created_at),
|
||||
updatedAt: new Date(user.updated_at)
|
||||
}
|
||||
});
|
||||
} catch (createUserError) {
|
||||
console.error('create user failed, skipping identities', user.email, createUserError);
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
console.log('backfilling accounts only for', user.email);
|
||||
}
|
||||
|
||||
for (const identity of user.identities) {
|
||||
const identities = parseIdentities(user.identities);
|
||||
|
||||
for (const identity of identities) {
|
||||
const existingAccounts = await ctx.internalAdapter.findAccounts(user.id);
|
||||
|
||||
if (identity.provider === 'email') {
|
||||
console.log('email provider');
|
||||
const hasCredential = existingAccounts.find((account) => account.providerId === 'credential');
|
||||
const hasCredential = existingAccounts.find((accountRow) => accountRow.providerId === 'credential');
|
||||
console.log('hasCredential', hasCredential);
|
||||
if (!hasCredential) {
|
||||
await ctx.adapter
|
||||
@@ -198,7 +356,7 @@ const migrateFromSupabase = async () => {
|
||||
}
|
||||
const supportedProviders = Object.keys(ctx.options.socialProviders || {});
|
||||
if (supportedProviders.includes(identity.provider)) {
|
||||
const hasAccount = existingAccounts.find((account) => account.providerId === identity.provider);
|
||||
const hasAccount = existingAccounts.find((accountRow) => accountRow.providerId === identity.provider);
|
||||
if (!hasAccount) {
|
||||
await ctx.adapter.create({
|
||||
model: 'account',
|
||||
@@ -216,6 +374,13 @@ const migrateFromSupabase = async () => {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (effectiveReset) {
|
||||
await restoreProfileUserFkey(db);
|
||||
}
|
||||
};
|
||||
|
||||
migrateFromSupabase();
|
||||
migrateFromSupabase().catch((migrationError) => {
|
||||
console.error('migrateFromSupabase failed:', migrationError);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user