Skip to content

Commit 64d0e97

Browse files
codercatdevcodercatdev
committed
try new logic for youtube views
1 parent 1b3189a commit 64d0e97

1 file changed

Lines changed: 172 additions & 66 deletions

File tree

‎app/api/youtube/views/route.tsx‎

Lines changed: 172 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -9,20 +9,21 @@ const sanityWriteClient = createClient({
99
projectId: process.env.NEXT_PUBLIC_SANITY_PROJECT_ID,
1010
dataset: process.env.NEXT_PUBLIC_SANITY_DATASET,
1111
token: process.env.SANITY_API_WRITE_TOKEN,
12-
apiVersion: "2022-03-07",
12+
apiVersion: "2025-09-15",
1313
perspective: "published",
1414
useCdn: false,
1515
});
1616

17-
18-
19-
2017
async function processBatchTasks() {
21-
// Fetch up to 50 pending tasks
18+
console.log("[YOUTUBE] Fetching up to 10 pending youtubeUpdateTask tasks from Sanity");
2219
let tasks = await sanityWriteClient.fetch(
23-
`*[_type == "youtubeUpdateTask" && status == "pending"]| order(lastChecked asc)[0...50]{ _id, targetDoc->{_id, _type, youtube}, status }`
20+
`*[_type == "youtubeUpdateTask" && status == "pending"]| order(lastChecked asc)[0...10]{ _id, targetDoc->{_id, _type, youtube}, status }`
2421
);
25-
if (!tasks || tasks.length === 0) return { processed: 0 };
22+
console.log(`[YOUTUBE] Fetched ${tasks?.length || 0} tasks`);
23+
if (!tasks || tasks.length === 0) {
24+
console.log("[YOUTUBE] No pending tasks found");
25+
return { processed: 0 };
26+
}
2627

2728
// Prepare video IDs and map taskId to docId
2829
const validTasks = [];
@@ -40,68 +41,150 @@ async function processBatchTasks() {
4041
}
4142
validTasks.push({ taskId, docId: targetDoc._id, youtubeId: id });
4243
}
44+
console.log(`[YOUTUBE] validTasks: ${validTasks.length}, errorTasks: ${errorTasks.length}`);
4345

44-
// Mark all valid tasks as inProgress
45-
for (const t of validTasks) {
46-
await sanityWriteClient.patch(t.taskId)
47-
.set({ status: "inProgress", lastChecked: new Date().toISOString() })
48-
.commit();
46+
// Mark all valid tasks as inProgress (sequential)
47+
try {
48+
for (const t of validTasks) {
49+
console.log(`[YOUTUBE] Marking task ${t.taskId} as inProgress`);
50+
try {
51+
await sanityWriteClient.patch(t.taskId)
52+
.set({ status: "inProgress", lastChecked: new Date().toISOString() })
53+
.commit({ visibility: "async" });
54+
console.log(`[YOUTUBE] Task ${t.taskId} marked as inProgress`);
55+
} catch (err) {
56+
console.error(`[YOUTUBE] Error marking task ${t.taskId} as inProgress:`, err);
57+
}
58+
}
59+
console.log(`[YOUTUBE] Marked ${validTasks.length} tasks as inProgress`);
60+
} catch (err) {
61+
console.error("[YOUTUBE] Error marking tasks as inProgress", err);
4962
}
5063

51-
// Mark all error tasks as error
52-
for (const t of errorTasks) {
53-
await sanityWriteClient.patch(t.taskId)
54-
.set({ status: "error", errorMessage: t.error, lastChecked: new Date().toISOString() })
55-
.commit();
64+
// Mark all error tasks as error (sequential)
65+
if (errorTasks.length > 0) {
66+
try {
67+
for (const t of errorTasks) {
68+
console.log(`[YOUTUBE] Marking error task ${t.taskId} as error: ${t.error}`);
69+
try {
70+
await sanityWriteClient.patch(t.taskId)
71+
.set({ status: "error", errorMessage: t.error, lastChecked: new Date().toISOString() })
72+
.commit({ visibility: "async" });
73+
console.log(`[YOUTUBE] Error task ${t.taskId} marked as error`);
74+
} catch (err) {
75+
console.error(`[YOUTUBE] Error marking error task ${t.taskId}:`, err);
76+
}
77+
}
78+
console.log(`[YOUTUBE] Marked ${errorTasks.length} tasks as error`);
79+
} catch (err) {
80+
console.error("[YOUTUBE] Error marking error tasks", err);
81+
}
5682
}
5783

58-
if (validTasks.length === 0) return { processed: 0, errors: errorTasks.length };
84+
if (validTasks.length === 0) {
85+
console.log("[YOUTUBE] No valid tasks to process");
86+
return { processed: 0, errors: errorTasks.length };
87+
}
5988

60-
// Batch YouTube API call
61-
const ids = validTasks.map(t => t.youtubeId).join(",");
62-
console.log("[YOUTUBE] Fetching stats for IDs:", ids);
63-
const videoResp = await fetch(
64-
`https://www.googleapis.com/youtube/v3/videos?id=${ids}&key=${process.env.YOUTUBE_API_KEY}&fields=items(id,statistics)&part=statistics`,
65-
);
66-
const json = await videoResp.json();
67-
if (videoResp.status !== 200) {
68-
// Mark all as error
69-
for (const t of validTasks) {
70-
await sanityWriteClient.patch(t.taskId)
71-
.set({ status: "error", errorMessage: JSON.stringify(json), lastChecked: new Date().toISOString() })
72-
.commit();
89+
// Batch YouTube API call in groups of 50 IDs
90+
const statsMap = new Map();
91+
const batchSize = 50;
92+
let apiError = null;
93+
for (let i = 0; i < validTasks.length; i += batchSize) {
94+
const batch = validTasks.slice(i, i + batchSize);
95+
const ids = batch.map(t => t.youtubeId).join(",");
96+
console.log(`[YOUTUBE] Fetching stats for IDs batch: ${ids}`);
97+
let videoResp: Response, json: any;
98+
try {
99+
videoResp = await fetch(
100+
`https://www.googleapis.com/youtube/v3/videos?id=${ids}&key=${process.env.YOUTUBE_API_KEY}&fields=items(id,statistics)&part=statistics`,
101+
);
102+
json = await videoResp.json();
103+
console.log(`[YOUTUBE] YouTube API response status: ${videoResp.status}`);
104+
if (videoResp.status !== 200) {
105+
console.error("[YOUTUBE] YouTube API error", json);
106+
apiError = json;
107+
// Mark all tasks in this batch as error
108+
for (const t of batch) {
109+
console.log(`[YOUTUBE] Marking batch task ${t.taskId} as error due to API error`);
110+
try {
111+
await sanityWriteClient.patch(t.taskId)
112+
.set({ status: "error", errorMessage: JSON.stringify(json), lastChecked: new Date().toISOString() })
113+
.commit({ visibility: "async" });
114+
console.log(`[YOUTUBE] Batch task ${t.taskId} marked as error`);
115+
} catch (err) {
116+
console.error(`[YOUTUBE] Error marking task ${t.taskId} as error after YouTube API error:`, err);
117+
}
118+
}
119+
break;
120+
}
121+
for (const item of json?.items || []) {
122+
console.log(`[YOUTUBE] Adding stats for video ID: ${item.id}`);
123+
statsMap.set(item.id, item.statistics);
124+
}
125+
} catch (err) {
126+
console.error("[YOUTUBE] Error fetching YouTube stats", err);
127+
apiError = err;
128+
for (const t of batch) {
129+
console.log(`[YOUTUBE] Marking batch task ${t.taskId} as error due to fetch error`);
130+
try {
131+
await sanityWriteClient.patch(t.taskId)
132+
.set({ status: "error", errorMessage: String(err), lastChecked: new Date().toISOString() })
133+
.commit({ visibility: "async" });
134+
console.log(`[YOUTUBE] Batch task ${t.taskId} marked as error`);
135+
} catch (err2) {
136+
console.error(`[YOUTUBE] Error marking task ${t.taskId} as error after fetch error:`, err2);
137+
}
138+
}
139+
break;
73140
}
74-
return { processed: 0, errors: validTasks.length };
75141
}
76-
const statsMap = new Map();
77-
for (const item of json?.items || []) {
78-
statsMap.set(item.id, item.statistics);
142+
console.log(`[YOUTUBE] StatsMap size: ${statsMap.size}`);
143+
if (apiError) {
144+
return { processed: 0, errors: validTasks.length };
79145
}
80146

81147
let completed = 0;
148+
const patchOps = [];
149+
const completedTaskOps = [];
150+
const erroredTaskOps = [];
82151
for (const t of validTasks) {
152+
console.log(`[YOUTUBE] Processing validTask: taskId=${t.taskId}, docId=${t.docId}, youtubeId=${t.youtubeId}`);
83153
const statistics = statsMap.get(t.youtubeId);
84154
if (!statistics) {
85-
await sanityWriteClient.patch(t.taskId)
86-
.set({ status: "error", errorMessage: "No statistics found", lastChecked: new Date().toISOString() })
87-
.commit();
155+
console.log(`[YOUTUBE] No statistics found for youtubeId=${t.youtubeId}, marking task ${t.taskId} as error`);
156+
try {
157+
await sanityWriteClient.patch(t.taskId)
158+
.set({ status: "error", errorMessage: "No statistics found", lastChecked: new Date().toISOString() })
159+
.commit({ visibility: "async" });
160+
console.log(`[YOUTUBE] Task ${t.taskId} marked as error (no statistics)`);
161+
} catch (err) {
162+
console.error(`[YOUTUBE] Error marking task ${t.taskId} as error:`, err);
163+
}
88164
continue;
89165
}
90-
// Update target doc with stats
91-
await sanityWriteClient.patch(t.docId)
92-
.set({
93-
"statistics.youtube.commentCount": Number.parseInt(statistics.commentCount),
94-
"statistics.youtube.favoriteCount": Number.parseInt(statistics.favoriteCount),
95-
"statistics.youtube.likeCount": Number.parseInt(statistics.likeCount),
96-
"statistics.youtube.viewCount": Number.parseInt(statistics.viewCount),
97-
})
98-
.commit();
99-
// Mark task as completed
100-
await sanityWriteClient.patch(t.taskId)
101-
.set({ status: "completed", lastChecked: new Date().toISOString(), errorMessage: undefined })
102-
.commit();
103-
completed++;
166+
try {
167+
console.log(`[YOUTUBE] Updating doc ${t.docId} with statistics for youtubeId=${t.youtubeId}`);
168+
await sanityWriteClient.patch(t.docId)
169+
.set({
170+
"statistics.youtube.commentCount": Number.parseInt(statistics.commentCount),
171+
"statistics.youtube.favoriteCount": Number.parseInt(statistics.favoriteCount),
172+
"statistics.youtube.likeCount": Number.parseInt(statistics.likeCount),
173+
"statistics.youtube.viewCount": Number.parseInt(statistics.viewCount),
174+
})
175+
.commit({ visibility: "async" });
176+
console.log(`[YOUTUBE] Updated doc ${t.docId}`);
177+
await sanityWriteClient.patch(t.taskId)
178+
.set({ status: "completed", lastChecked: new Date().toISOString(), errorMessage: undefined })
179+
.commit({ visibility: "async" });
180+
console.log(`[YOUTUBE] Task ${t.taskId} marked as completed`);
181+
completed++;
182+
} catch (err) {
183+
console.error(`[YOUTUBE] Error updating doc ${t.docId} or task ${t.taskId}:`, err);
184+
}
104185
}
186+
// Log summary
187+
console.log(`[YOUTUBE] Patched ${completed} docs/tasks sequentially`);
105188
return { processed: completed, errors: errorTasks.length + (validTasks.length - completed) };
106189
}
107190

@@ -113,36 +196,59 @@ export async function POST(request: NextRequest) {
113196
}
114197

115198
try {
199+
console.log("[YOUTUBE] POST handler started");
116200
// Repopulate youtubeUpdateTask queue if empty
117201
let tasks = await sanityWriteClient.fetch(
118202
`*[_type == "youtubeUpdateTask" && status == "pending"]| order(lastChecked asc)[0...1]{ _id }`
119203
);
204+
console.log(`[YOUTUBE] Pending tasks in queue: ${tasks?.length || 0}`);
120205
if (!tasks || tasks.length === 0) {
121-
const posts = await sanityWriteClient.fetch(
122-
'*[_type == "post" && defined(youtube)]{_id, _type, youtube}'
123-
);
206+
console.log("[YOUTUBE] No pending tasks, repopulating queue from posts and podcasts");
124207
const podcasts = await sanityWriteClient.fetch(
125-
'*[_type == "podcast" && defined(youtube)]{_id, _type, youtube}'
208+
'*[_type == "podcast" && defined(youtube)]{_id, _type, youtube,date} | order(date desc)[0...1000]'
209+
);
210+
const posts = await sanityWriteClient.fetch(
211+
'*[_type == "post" && defined(youtube)]{_id, _type, youtube,date} | order(date desc)[0...1000]'
126212
);
127-
const allDocs = [...posts, ...podcasts];
213+
const allDocs = [...podcasts, ...posts];
214+
console.log(`[YOUTUBE] Found ${allDocs.length} docs with YouTube`);
215+
// Fetch all existing youtubeUpdateTask docs for these docs
128216
const existingTasks = await sanityWriteClient.fetch(
129-
'*[_type == "youtubeUpdateTask" && defined(targetDoc._ref)]{targetDoc}'
217+
'*[_type == "youtubeUpdateTask" && defined(targetDoc._ref)]{_id, targetDoc}'
130218
);
131-
const existingIds = new Set(existingTasks.map((t: { targetDoc?: { _ref?: string } }) => t.targetDoc?._ref));
219+
const existingTaskMap = new Map(existingTasks.map((t: { _id: string, targetDoc?: { _ref?: string } }) => [t.targetDoc?._ref, t._id]));
132220
for (const doc of allDocs) {
133-
if (!existingIds.has(doc._id)) {
134-
await sanityWriteClient.create({
135-
_type: "youtubeUpdateTask",
136-
targetDoc: { _type: "reference", _ref: doc._id },
137-
status: "pending",
138-
lastChecked: null,
139-
});
221+
const taskId = existingTaskMap.get(doc._id);
222+
if (typeof taskId === 'string' && taskId) {
223+
// Update status to pending
224+
try {
225+
await sanityWriteClient.patch(taskId)
226+
.set({ status: "pending", lastChecked: null })
227+
.commit({ visibility: "async" });
228+
console.log(`[YOUTUBE] Marked existing youtubeUpdateTask ${taskId} as pending for doc ${doc._id}`);
229+
} catch (err) {
230+
console.error(`[YOUTUBE] Error marking youtubeUpdateTask ${taskId} as pending for doc ${doc._id}:`, err);
231+
}
232+
} else {
233+
// Create new youtubeUpdateTask
234+
try {
235+
await sanityWriteClient.create({
236+
_type: "youtubeUpdateTask",
237+
targetDoc: { _type: "reference", _ref: doc._id },
238+
status: "pending",
239+
lastChecked: null,
240+
});
241+
console.log(`[YOUTUBE] Created new youtubeUpdateTask for doc ${doc._id}`);
242+
} catch (err) {
243+
console.error(`[YOUTUBE] Error creating youtubeUpdateTask for doc ${doc._id}:`, err);
244+
}
140245
}
141246
}
142247
}
143248

144249
// Process a batch of tasks
145250
const result = await processBatchTasks();
251+
console.log("[YOUTUBE] Batch process result", result);
146252
return Response.json({ success: true, ...result });
147253
} catch (error) {
148254
console.error("[YOUTUBE] Unexpected error:", error);

0 commit comments

Comments
 (0)