dredgegram/playwright/scrape-story-content.spec.js

import { test } from '@playwright/test';
import fs from 'fs';
import path from 'path';

test.use({ storageState: path.resolve(__dirname, '.auth', 'ig-session.json') });

function deepFind(obj, targetKey) {
    if (obj === null || typeof obj !== 'object') return null;
    if (targetKey in obj) return obj[targetKey];
    for (const value of Object.values(obj)) {
        const found = deepFind(value, targetKey);
        if (found) return found;
    }
    return null;
}

function extractStoryContent(node) {
    return {
        user_id: node.id,
        username: node.user?.username,
        profile_pic_url: node.user?.profile_pic_url || null,
        seen: node.seen,
        latest_reel_media: node.latest_reel_media,
        muted: node.muted,
        items: (node.items || []).map((item) => ({
            id: item.pk,
            code: item.code,
            is_video: item.media_type === 2,
            taken_at: item.taken_at,
            expiring_at: item.expiring_at,
            image_url: item.image_versions2?.candidates?.[0]?.url || null,
            video_url: item.video_versions?.[0]?.url || null,
            accessibility_caption: item.accessibility_caption || null,
        })),
    };
}

test('fetch all story content directly', async ({ page }) => {
    test.setTimeout(60_000);

    // Load the page first so we have valid csrf/session context in cookies
    await page.goto('https://www.instagram.com/');
    await page.waitForTimeout(3000);

    // Pull the tray's user IDs the same way as your existing stories scrape
    const reelIds = await page.evaluate(() => {
        function deepFind(obj, targetKey) {
            if (obj === null || typeof obj !== 'object') return null;
            if (targetKey in obj) return obj[targetKey];
            for (const value of Object.values(obj)) {
                const found = deepFind(value, targetKey);
                if (found) return found;
            }
            return null;
        }
        const scripts = document.querySelectorAll(
            'script[type="application/json"]',
        );
        for (const script of scripts) {
            if (script.textContent.includes('xdt_api__v1__feed__reels_tray')) {
                const json = JSON.parse(script.textContent);
                const reelsTray = deepFind(
                    json,
                    'xdt_api__v1__feed__reels_tray',
                );
                return (reelsTray?.tray || []).map((item) => item.id);
            }
        }
        return [];
    });

    if (reelIds.length === 0) {
        console.log('No reel IDs found in tray - nothing to fetch');
        return;
    }

    // Get the csrf token from cookies (needed as a header for the POST)
    const cookies = await page.context().cookies();
    const csrfToken = cookies.find((c) => c.name === 'csrftoken')?.value;

    const variables = {
        initial_reel_id: reelIds[0],
        reel_ids: reelIds,
        first: reelIds.length,
        last: 0,
        __relay_internal__pv__PolarisCommunityNoteStoriesLabelEnabledrelayprovider: true,
        __relay_internal__pv__PolarisAIGMMediaWebLabelEnabledrelayprovider: false,
    };

    const body = new URLSearchParams({
        fb_api_req_friendly_name: 'PolarisStoriesV3ReelPageGalleryQuery',
        variables: JSON.stringify(variables),
        doc_id: '27768924676024297',
        server_timestamps: 'true',
    });

    const response = await page.request.post(
        'https://www.instagram.com/graphql/query',
        {
            headers: {
                'content-type': 'application/x-www-form-urlencoded',
                'x-csrftoken': csrfToken,
                'x-ig-app-id': '936619743392459',
            },
            data: body.toString(),
        },
    );

    const json = await response.json();
    const edges =
        json?.data?.xdt_api__v1__feed__reels_media__connection?.edges || [];
    const storyContent = edges.map((edge) => extractStoryContent(edge.node));

    const storyItems = storyContent.flatMap((story) =>
        story.items.map((item) => ({
            id: item.id,
            user_id: story.user_id,
            username: story.username,
            profile_pic_url: story.profile_pic_url,
            code: item.code,
            is_video: item.is_video,
            taken_at: item.taken_at,
            expiring_at: item.expiring_at,
            image_url: item.image_url,
            video_url: item.video_url,
            accessibility_caption: item.accessibility_caption,
            muted: story.muted,
        })),
    );

    const outputPath = path.resolve(__dirname, 'output', 'scrape-stories.json');
    fs.mkdirSync(path.dirname(outputPath), { recursive: true });
    fs.writeFileSync(outputPath, JSON.stringify(storyItems));
});