1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
| import { NextRequest, NextResponse } from 'next/server';
import { Readability } from '@mozilla/readability';
import { JSDOM } from 'jsdom';
// @ts-ignore
import TurndownService from 'turndown';
import { createClient } from 'webdav';
import iconv from 'iconv-lite';
const TELEGRAM_TOKEN = process.env.TELEGRAM_TOKEN;
const KOOFR_EMAIL = process.env.KOOFR_EMAIL;
const KOOFR_APP_PASSWORD = process.env.KOOFR_APP_PASSWORD;
const ALLOWED_USER_ID = process.env.ALLOWED_USER_ID;
const KOOFR_WEBDAV_URL = 'https://app.koofr.net/dav/Koofr';
// jsdom, iconv-lite, webdav는 Node.js API를 사용하므로 Edge Runtime을 사용하지 않는다.
export const runtime = 'nodejs';
const USER_AGENT =
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' +
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36';
/**
* HTML 앞부분은 charset 선언 자체가 ASCII이므로 latin1로 읽어도 안전하다.
* meta charset과 과거형 http-equiv/content 선언을 모두 지원한다.
*/
function getDeclaredCharset(buffer: Buffer, contentType: string | null): string | null {
const headerMatch = contentType?.match(/charset\s*=\s*["']?([^\s;"']+)/i);
if (headerMatch) return headerMatch[1].toLowerCase();
const head = buffer.subarray(0, 64 * 1024).toString('latin1');
const metaCharset = head.match(/<meta\b[^>]*\bcharset\s*=\s*["']?([^\s"'/>;]+)/i);
if (metaCharset) return metaCharset[1].toLowerCase();
const httpEquiv = head.match(
/<meta\b[^>]*\bcontent\s*=\s*["'][^"']*charset\s*=\s*([^\s;"']+)[^"']*["'][^>]*>/i,
);
return httpEquiv?.[1]?.toLowerCase() ?? null;
}
function normalizeCharset(charset: string | null): string | null {
if (!charset) return null;
const value = charset.trim().toLowerCase().replace(/["']/g, '');
if (['utf-8', 'utf8'].includes(value)) return 'utf-8';
if (
value.includes('euc-kr') ||
value.includes('cp949') ||
value.includes('ms949') ||
value.includes('x-windows-949') ||
value.includes('ks_c_5601') ||
value.includes('ks-c-5601')
) {
// CP949는 EUC-KR의 확장 문자까지 포함하므로 국내 레거시 사이트에 더 안전하다.
return 'cp949';
}
return iconv.encodingExists(value) ? value : null;
}
function isValidUtf8(buffer: Buffer): boolean {
try {
new TextDecoder('utf-8', { fatal: true }).decode(buffer);
return true;
} catch {
return false;
}
}
function countBrokenCharacters(value: string): number {
return (value.match(/\uFFFD/g) || []).length + (value.match(/[\u0000-\u0008\u000B\u000C\u000E-\u001F]/g) || []).length;
}
/**
* 최신 네이버처럼 선언값과 실제 바이트가 어긋나는 경우를 막기 위해
* 유효한 UTF-8 바이트는 UTF-8을 최우선으로 사용한다.
* UTF-8이 아니면 선언 인코딩과 CP949 후보 중 손상 문자가 적은 결과를 택한다.
*/
function decodeHtml(buffer: Buffer, contentType: string | null): string {
// UTF-8 BOM
if (buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) {
return iconv.decode(buffer, 'utf-8');
}
if (isValidUtf8(buffer)) return iconv.decode(buffer, 'utf-8');
const declared = normalizeCharset(getDeclaredCharset(buffer, contentType));
const encodings = Array.from(new Set([declared, 'cp949'].filter(Boolean))) as string[];
const candidates = encodings.map(encoding => ({
encoding,
html: iconv.decode(buffer, encoding),
}));
candidates.sort((a, b) => countBrokenCharacters(a.html) - countBrokenCharacters(b.html));
return candidates[0]?.html ?? iconv.decode(buffer, 'utf-8');
}
function normalizeText(value: string | null | undefined): string {
return (value || '')
.normalize('NFC')
.replace(/\uFFFD+/g, '')
.replace(/[\u0000-\u001F\u007F]/g, ' ')
.replace(/\s+/g, ' ')
.trim();
}
function yamlString(value: string): string {
return JSON.stringify(normalizeText(value));
}
function makeSafeFileName(title: string): string {
const cleaned = normalizeText(title)
.replace(/[\\/:*?"<>|%]/g, '-')
.replace(/\.+$/g, '')
.replace(/\s+/g, ' ')
.trim()
.slice(0, 150);
return `${cleaned || 'Untitled'}.md`;
}
export async function POST(req: NextRequest) {
try {
const body = await req.json();
const message = body.message;
if (!message?.text) return NextResponse.json({ ok: true });
const chatId = message.chat.id;
if (ALLOWED_USER_ID && String(chatId) !== String(ALLOWED_USER_ID)) {
return NextResponse.json({ ok: true });
}
const urlMatch = message.text.match(/https?:\/\/[^\s]+/g);
if (!urlMatch) return NextResponse.json({ ok: true });
let targetUrl = urlMatch[0];
try {
const urlObj = new URL(targetUrl);
['utm_source', 'utm_medium', 'utm_campaign', 'ref'].forEach(param =>
urlObj.searchParams.delete(param),
);
targetUrl = urlObj.toString();
} catch {
// URL 생성 실패 시 Telegram에서 추출한 원문을 그대로 사용한다.
}
await sendTelegramMessage(chatId, '🔍 원본 콘텐츠 및 고화질 이미지 분석 중...');
try {
const response = await fetch(targetUrl, {
redirect: 'follow',
headers: {
'User-Agent': USER_AGENT,
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'ko-KR,ko;q=0.9,en-US;q=0.7,en;q=0.5',
},
});
if (!response.ok) {
throw new Error(`HTTP ${response.status} ${response.statusText}`);
}
const contentType = response.headers.get('content-type');
const buffer = Buffer.from(await response.arrayBuffer());
const html = decodeHtml(buffer, contentType);
// naver.me 등 단축 URL은 최종 리다이렉트 주소를 base URL로 사용해야 한다.
const effectiveUrl = response.url || targetUrl;
const dom = new JSDOM(html, { url: effectiveUrl });
const document = dom.window.document;
// Readability가 DOM을 변경하기 전에 신뢰도 높은 제목 후보를 보관한다.
const ogTitle = normalizeText(
document.querySelector('meta[property="og:title"]')?.getAttribute('content'),
);
const originalDocumentTitle = normalizeText(document.title);
['script', 'style', 'noscript', 'footer', 'nav'].forEach(selector => {
document.querySelectorAll(selector).forEach(element => element.remove());
});
// 본문 iframe을 무조건 제거하면 일부 네이버 카페 페이지의 실제 글도 사라질 수 있다.
// 광고/추적용 iframe만 제거하고, 같은 네이버 계열 iframe은 남긴다.
document.querySelectorAll('iframe').forEach(iframe => {
const src = iframe.getAttribute('src') || '';
let keep = false;
try {
const host = new URL(src, effectiveUrl).hostname;
keep = host === 'naver.com' || host.endsWith('.naver.com');
} catch {
keep = false;
}
if (!keep) iframe.remove();
});
document.querySelectorAll('a').forEach(anchor => {
const href = anchor.getAttribute('href');
if (!href || href === '#' || anchor.querySelector('img')) {
anchor.replaceWith(...Array.from(anchor.childNodes));
}
});
const article = new Readability(document).parse();
if (!article?.content) {
await sendTelegramMessage(chatId, '❌ 본문을 추출할 수 없습니다.');
return NextResponse.json({ ok: true });
}
const readabilityTitle = normalizeText(article.title);
const title = ogTitle || readabilityTitle || originalDocumentTitle || 'Untitled';
const turndownService = new TurndownService({
headingStyle: 'atx',
hr: '---',
bulletListMarker: '-',
codeBlockStyle: 'fenced',
});
turndownService.addRule('absoluteImages', {
filter: 'img',
replacement: function (_content, node: any) {
const src =
node.getAttribute('data-lazy-src') ||
node.getAttribute('data-source') ||
node.getAttribute('data-src') ||
node.getAttribute('src');
if (!src) return '';
try {
let absoluteUrl = new URL(src.split(' ')[0], effectiveUrl).href;
if (absoluteUrl.includes('pstatic.net') || absoluteUrl.includes('blogfiles')) {
const cleanUrl = new URL(absoluteUrl);
if (cleanUrl.searchParams.has('type')) cleanUrl.searchParams.set('type', 'w1');
absoluteUrl = cleanUrl.toString();
}
const alt = normalizeText(node.getAttribute('alt')) || 'image';
return `\n![${alt.replace(/[\[\]]/g, '')}](${absoluteUrl})\n`;
} catch {
return '';
}
},
});
const markdownContent = turndownService.turndown(article.content);
const now = new Date();
const description = normalizeText(article.excerpt);
const author = normalizeText(article.byline) || 'Unknown';
const frontmatter = `---
title: ${yamlString(title)}
description: ${yamlString(description)}
source: ${yamlString(effectiveUrl)}
author: ${yamlString(author)}
created: ${now.toISOString().split('T')[0]}
scraped_at: ${yamlString(now.toLocaleString('ko-KR', { timeZone: 'Asia/Seoul' }))}
tags: ["ReadItLater"]
---
`;
const finalContent = `${frontmatter}# ${title}\n\n${markdownContent}`;
const fileName = makeSafeFileName(title);
const client = createClient(KOOFR_WEBDAV_URL, {
username: KOOFR_EMAIL,
password: KOOFR_APP_PASSWORD,
});
await client.putFileContents(`/ReadItLater/${fileName}`, finalContent, {
overwrite: true,
});
await sendTelegramMessage(chatId, `✅ 아카이빙 완료!\n\n📄 ${fileName}`, true);
} catch (error) {
console.error('Error:', error);
await sendTelegramMessage(chatId, '❌ 처리 중 에러가 발생했습니다.');
}
return NextResponse.json({ ok: true });
} catch (error) {
console.error('Webhook error:', error);
return NextResponse.json({ ok: false }, { status: 500 });
}
}
async function sendTelegramMessage(chatId: number, text: string, disablePreview = false) {
const url = `https://api.telegram.org/bot${TELEGRAM_TOKEN}/sendMessage`;
await fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
chat_id: chatId,
text,
disable_web_page_preview: disablePreview,
}),
});
}
|
