dredgegram/playwright/scrape-messages.spec.js

import { test } from '@playwright/test';
import fs from 'fs';
import path from 'path';

test.use({ storageState: path.resolve(__dirname, '.auth', 'ig-session.json') });

function randomWait(min = 2500, max = 5000) {
    return Math.floor(Math.random() * (max - min + 1)) + min;
}

function extractThread(t) {
    const isGroup =
        (t.thread_subtype || '').includes('GROUP') ||
        (t.users || []).length > 1;
    return {
        thread_id: t.id,
        thread_key: t.thread_key,
        thread_title: t.thread_title || null,
        is_group: isGroup,
        participants: (t.users || []).map((u) => ({
            user_id: u.id,
            username: u.username,
            full_name: u.full_name,
            profile_pic_url: u.profile_pic_url || null,
            interop_messaging_user_fbid: u.interop_messaging_user_fbid,
        })),
        last_activity_at: t.last_activity_timestamp_ms
            ? Number(t.last_activity_timestamp_ms)
            : null,
        muted: t.is_muted ?? null,
    };
}

// Inbox-preview messages only carry sender_fbid; thread-detail messages carry
// a full sender.user_dict. Build a lookup so preview messages, and reactions
// (which are keyed by sender_fbid too), still resolve to a username.
function buildFbidLookup(t) {
    const lookup = {};
    for (const u of t.users || []) {
        lookup[u.interop_messaging_user_fbid] = {
            user_id: u.id,
            username: u.username,
            full_name: u.full_name,
        };
    }
    return lookup;
}

function extractShare(node) {
    const xma = node.content?.xma;
    if (!xma) return null;
    return {
        url: xma.target_url || null,
        preview_image: xma.preview_image?.url || xma.header_icon?.url || null,
        title: xma.header_title_text || xma.title_text || null,
        caption:
            xma.caption_body_text ||
            xma.eyebrow_text ||
            xma.header_subtitle_text ||
            null,
    };
}

function extractMessage(node, thread, fbidLookup) {
    const senderDict = node.sender?.user_dict;
    const resolved = senderDict
        ? {
              user_id: node.sender.igid,
              username: senderDict.username,
              full_name: senderDict.full_name,
          }
        : fbidLookup[node.sender_fbid] || {};
    const reactions = (node.reactions || []).map((r) => ({
        emoji: r.reaction,
        sender_fbid: r.sender_fbid,
        username: fbidLookup[r.sender_fbid]?.username || null,
    }));
    return {
        id: node.id || node.message_id,
        thread_id: thread.id,
        user_id: resolved.user_id || null,
        username: resolved.username || null,
        full_name: resolved.full_name || null,
        content_type: node.content_type,
        text: node.text_body || node.content?.text_body || null,
        timestamp_ms: node.timestamp_ms ? Number(node.timestamp_ms) : null,
        share: extractShare(node),
        reactions: reactions.length ? reactions : null,
        replied_to_id: node.replied_to_message_id || null,
        replied_to_text: node.replied_to_message?.text_body || null,
        hasDetail: !!senderDict,
    };
}

test('fetch all direct message content', async ({ page }) => {
    test.setTimeout(180_000);

    const threadsMap = new Map();
    const messagesMap = new Map();

    function ingestThread(t) {
        if (!t?.id) return;
        threadsMap.set(t.id, extractThread(t));
        const fbidLookup = buildFbidLookup(t);
        for (const edge of t.slide_messages?.edges || []) {
            const msg = extractMessage(edge.node, t, fbidLookup);
            const existing = messagesMap.get(msg.id);
            if (existing && existing.hasDetail && !msg.hasDetail) continue; // don't clobber a richer entry
            messagesMap.set(msg.id, msg);
        }
    }

    page.on('response', async (response) => {
        const url = response.url();
        if (!url.includes('/api/graphql')) return;
        const params = new URLSearchParams(response.request().postData() || '');
        const friendlyName = params.get('fb_api_req_friendly_name');
        if (
            friendlyName !== 'PolarisDirectInboxQuery' &&
            friendlyName !== 'IGDThreadDetailQuery'
        )
            return;
        try {
            const json = await response.json();
            if (friendlyName === 'PolarisDirectInboxQuery') {
                const edges =
                    json?.data?.get_slide_mailbox_for_iris_subscription
                        ?.threads_by_folder?.edges || [];
                for (const edge of edges)
                    ingestThread(edge.node?.as_ig_direct_thread);
            } else {
                ingestThread(
                    json?.data?.get_slide_thread_nullable?.as_ig_direct_thread,
                );
            }
        } catch (e) {
            /* not JSON, ignore */
        }
    });

    await page.goto('https://www.instagram.com/direct/inbox/');
    await page.waitForTimeout(6000);

    // Open each conversation so its fuller message history loads via
    // IGDThreadDetailQuery, using the real hrefs Instagram rendered rather
    // than guessing the thread-id format ourselves
    const threadHrefs = await page.$$eval('a[href*="/direct/t/"]', (els) => [
        ...new Set(els.map((el) => el.getAttribute('href'))),
    ]);

    for (const href of threadHrefs) {
        await page.goto(`https://www.instagram.com${href}`);
        await page.waitForTimeout(randomWait());
    }

    const outputPath = path.resolve(
        __dirname,
        'output',
        'scrape-messages.json',
    );
    fs.mkdirSync(path.dirname(outputPath), { recursive: true });
    fs.writeFileSync(
        outputPath,
        JSON.stringify({
            threads: [...threadsMap.values()],
            messages: [...messagesMap.values()].map(
                ({ hasDetail, ...msg }) => msg,
            ),
        }),
    );
});