Initial Commit
This commit is contained in:
@@ -0,0 +1,171 @@
|
||||
import { test } from '@playwright/test';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
test.use({ storageState: path.resolve(__dirname, '.auth', 'ig-session.json') });
|
||||
|
||||
function randomWait(min = 2500, max = 5000) {
|
||||
return Math.floor(Math.random() * (max - min + 1)) + min;
|
||||
}
|
||||
|
||||
function extractThread(t) {
|
||||
const isGroup =
|
||||
(t.thread_subtype || '').includes('GROUP') ||
|
||||
(t.users || []).length > 1;
|
||||
return {
|
||||
thread_id: t.id,
|
||||
thread_key: t.thread_key,
|
||||
thread_title: t.thread_title || null,
|
||||
is_group: isGroup,
|
||||
participants: (t.users || []).map((u) => ({
|
||||
user_id: u.id,
|
||||
username: u.username,
|
||||
full_name: u.full_name,
|
||||
profile_pic_url: u.profile_pic_url || null,
|
||||
interop_messaging_user_fbid: u.interop_messaging_user_fbid,
|
||||
})),
|
||||
last_activity_at: t.last_activity_timestamp_ms
|
||||
? Number(t.last_activity_timestamp_ms)
|
||||
: null,
|
||||
muted: t.is_muted ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
// Inbox-preview messages only carry sender_fbid; thread-detail messages carry
|
||||
// a full sender.user_dict. Build a lookup so preview messages, and reactions
|
||||
// (which are keyed by sender_fbid too), still resolve to a username.
|
||||
function buildFbidLookup(t) {
|
||||
const lookup = {};
|
||||
for (const u of t.users || []) {
|
||||
lookup[u.interop_messaging_user_fbid] = {
|
||||
user_id: u.id,
|
||||
username: u.username,
|
||||
full_name: u.full_name,
|
||||
};
|
||||
}
|
||||
return lookup;
|
||||
}
|
||||
|
||||
function extractShare(node) {
|
||||
const xma = node.content?.xma;
|
||||
if (!xma) return null;
|
||||
return {
|
||||
url: xma.target_url || null,
|
||||
preview_image: xma.preview_image?.url || xma.header_icon?.url || null,
|
||||
title: xma.header_title_text || xma.title_text || null,
|
||||
caption:
|
||||
xma.caption_body_text ||
|
||||
xma.eyebrow_text ||
|
||||
xma.header_subtitle_text ||
|
||||
null,
|
||||
};
|
||||
}
|
||||
|
||||
function extractMessage(node, thread, fbidLookup) {
|
||||
const senderDict = node.sender?.user_dict;
|
||||
const resolved = senderDict
|
||||
? {
|
||||
user_id: node.sender.igid,
|
||||
username: senderDict.username,
|
||||
full_name: senderDict.full_name,
|
||||
}
|
||||
: fbidLookup[node.sender_fbid] || {};
|
||||
const reactions = (node.reactions || []).map((r) => ({
|
||||
emoji: r.reaction,
|
||||
sender_fbid: r.sender_fbid,
|
||||
username: fbidLookup[r.sender_fbid]?.username || null,
|
||||
}));
|
||||
return {
|
||||
id: node.id || node.message_id,
|
||||
thread_id: thread.id,
|
||||
user_id: resolved.user_id || null,
|
||||
username: resolved.username || null,
|
||||
full_name: resolved.full_name || null,
|
||||
content_type: node.content_type,
|
||||
text: node.text_body || node.content?.text_body || null,
|
||||
timestamp_ms: node.timestamp_ms ? Number(node.timestamp_ms) : null,
|
||||
share: extractShare(node),
|
||||
reactions: reactions.length ? reactions : null,
|
||||
replied_to_id: node.replied_to_message_id || null,
|
||||
replied_to_text: node.replied_to_message?.text_body || null,
|
||||
hasDetail: !!senderDict,
|
||||
};
|
||||
}
|
||||
|
||||
test('fetch all direct message content', async ({ page }) => {
|
||||
test.setTimeout(180_000);
|
||||
|
||||
const threadsMap = new Map();
|
||||
const messagesMap = new Map();
|
||||
|
||||
function ingestThread(t) {
|
||||
if (!t?.id) return;
|
||||
threadsMap.set(t.id, extractThread(t));
|
||||
const fbidLookup = buildFbidLookup(t);
|
||||
for (const edge of t.slide_messages?.edges || []) {
|
||||
const msg = extractMessage(edge.node, t, fbidLookup);
|
||||
const existing = messagesMap.get(msg.id);
|
||||
if (existing && existing.hasDetail && !msg.hasDetail) continue; // don't clobber a richer entry
|
||||
messagesMap.set(msg.id, msg);
|
||||
}
|
||||
}
|
||||
|
||||
page.on('response', async (response) => {
|
||||
const url = response.url();
|
||||
if (!url.includes('/api/graphql')) return;
|
||||
const params = new URLSearchParams(response.request().postData() || '');
|
||||
const friendlyName = params.get('fb_api_req_friendly_name');
|
||||
if (
|
||||
friendlyName !== 'PolarisDirectInboxQuery' &&
|
||||
friendlyName !== 'IGDThreadDetailQuery'
|
||||
)
|
||||
return;
|
||||
try {
|
||||
const json = await response.json();
|
||||
if (friendlyName === 'PolarisDirectInboxQuery') {
|
||||
const edges =
|
||||
json?.data?.get_slide_mailbox_for_iris_subscription
|
||||
?.threads_by_folder?.edges || [];
|
||||
for (const edge of edges)
|
||||
ingestThread(edge.node?.as_ig_direct_thread);
|
||||
} else {
|
||||
ingestThread(
|
||||
json?.data?.get_slide_thread_nullable?.as_ig_direct_thread,
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
/* not JSON, ignore */
|
||||
}
|
||||
});
|
||||
|
||||
await page.goto('https://www.instagram.com/direct/inbox/');
|
||||
await page.waitForTimeout(6000);
|
||||
|
||||
// Open each conversation so its fuller message history loads via
|
||||
// IGDThreadDetailQuery, using the real hrefs Instagram rendered rather
|
||||
// than guessing the thread-id format ourselves
|
||||
const threadHrefs = await page.$$eval('a[href*="/direct/t/"]', (els) => [
|
||||
...new Set(els.map((el) => el.getAttribute('href'))),
|
||||
]);
|
||||
|
||||
for (const href of threadHrefs) {
|
||||
await page.goto(`https://www.instagram.com${href}`);
|
||||
await page.waitForTimeout(randomWait());
|
||||
}
|
||||
|
||||
const outputPath = path.resolve(
|
||||
__dirname,
|
||||
'output',
|
||||
'scrape-messages.json',
|
||||
);
|
||||
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
outputPath,
|
||||
JSON.stringify({
|
||||
threads: [...threadsMap.values()],
|
||||
messages: [...messagesMap.values()].map(
|
||||
({ hasDetail, ...msg }) => msg,
|
||||
),
|
||||
}),
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user