Initial Commit

2026-08-28 22:55:32 +10:00
parent d8d4ecbe8a
commit f00405ed24
16 changed files with 1008 additions and 0 deletions
@@ -0,0 +1,106 @@
+import { test } from '@playwright/test';
+import fs from 'fs';
+import path from 'path';
+
+test.use({ storageState: path.resolve(__dirname, '.auth', 'ig-session.json') });
+function randomWait(min = 2500, max = 5000) {
+    return Math.floor(Math.random() * (max - min + 1)) + min;
+}
+
+function extractPostsFromMedia(m) {
+    return {
+        id: m.pk,
+        code: m.code,
+        username: m.user?.username,
+        full_name: m.user?.full_name,
+        profile_pic_url:
+            m.user?.hd_profile_pic_url_info?.url ||
+            m.user?.profile_pic_url ||
+            null,
+        caption: m.caption?.text || null,
+        accessibility_caption: m.accessibility_caption || null,
+        is_video: m.media_type === 2,
+        is_carousel: m.media_type === 8,
+        like_count: m.like_count,
+        comment_count: m.comment_count,
+        taken_at: m.taken_at,
+        image_url: m.image_versions2?.candidates?.[0]?.url || null,
+        video_url: m.video_versions?.[0]?.url || null,
+        carousel_images:
+            m.carousel_media?.map((item) => ({
+                url: item.image_versions2?.candidates?.[0]?.url || null,
+                video_url: item.video_versions?.[0]?.url || null,
+                accessibility_caption: item.accessibility_caption || null,
+            })) || null,
+    };
+}
+
+test('scrape following feed', async ({ page }) => {
+    test.setTimeout(120_000);
+    const posts = [];
+
+    page.on('response', async (response) => {
+        const url = response.url();
+        if (!url.includes('/graphql/query')) return;
+        try {
+            const json = await response.json();
+            const edges =
+                json?.data?.xdt_api__v1__feed__timeline__connection?.edges;
+            if (!edges) return;
+            for (const edge of edges) {
+                const m = edge.node?.media;
+                if (m) posts.push(extractPostsFromMedia(m));
+            }
+        } catch (e) {
+            /* not JSON, ignore */
+        }
+    });
+
+    await page.goto('https://www.instagram.com/?variant=following&hl=en');
+    await page.waitForTimeout(4000);
+
+    const preloadedEdges = await page.evaluate(() => {
+        function deepFind(obj, targetKey) {
+            if (obj === null || typeof obj !== 'object') return null;
+            if (targetKey in obj) return obj[targetKey];
+            for (const value of Object.values(obj)) {
+                const found = deepFind(value, targetKey);
+                if (found) return found;
+            }
+            return null;
+        }
+
+        const results = [];
+        document
+            .querySelectorAll('script[type="application/json"]')
+            .forEach((script) => {
+                try {
+                    const json = JSON.parse(script.textContent);
+                    const connection = deepFind(
+                        json,
+                        'xdt_api__v1__feed__timeline__connection',
+                    );
+                    if (connection?.edges) {
+                        results.push(...connection.edges);
+                    }
+                } catch (e) {
+                    /* not parseable JSON, skip */
+                }
+            });
+        return results;
+    });
+
+    for (const edge of preloadedEdges) {
+        const m = edge.node?.media;
+        if (m) posts.push(extractPostsFromMedia(m));
+    }
+
+    for (let i = 0; i < 10; i++) {
+        await page.mouse.wheel(0, 3000);
+        await page.waitForTimeout(randomWait());
+    }
+
+    const outputPath = path.resolve(__dirname, 'output', 'scrape-feed.json');
+    fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+    fs.writeFileSync(outputPath, JSON.stringify(posts));
+});