Initial Commit

2026-08-28 22:55:32 +10:00
parent d8d4ecbe8a
commit f00405ed24
16 changed files with 1008 additions and 0 deletions
@@ -27,3 +27,7 @@ yarn-error.log
 /.nova
 /.vscode
 /.zed
+/playwright/node_modules
+playwright/.auth/
+playwright/output/
+playwright/test-results
@@ -0,0 +1,15 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Attributes\Description;
+use Illuminate\Console\Attributes\Signature;
+
+#[Signature('app:debug-playwright')]
+#[Description('Command description')]
+class DebugPlaywright extends RunsPlaywrightScript
+{
+    protected $signature = 'direct:debug';
+    protected $description = 'Run the Playwright network/DOM debugging script against Instagram Direct inbox';
+    protected string $script = 'debugger.spec.js';
+}
@@ -0,0 +1,38 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Process;
+
+abstract class RunsPlaywrightScript extends Command
+{
+    protected string $script;
+    protected int $processTimeout = 300;
+
+    public function handle(): int
+    {
+        $this->info("Running {$this->script}...");
+
+        $result = Process::path(base_path('playwright'))
+            ->timeout($this->processTimeout)
+            ->run("npx playwright test {$this->script} --reporter=line");
+
+        $this->line($result->output());
+
+        if (! $result->successful()) {
+            $this->error($result->errorOutput());
+            return self::FAILURE;
+        }
+
+        $this->afterRun();
+
+        $this->info('Done.');
+        return self::SUCCESS;
+    }
+
+    protected function afterRun(): void
+    {
+        // no-op by default
+    }
+}
@@ -0,0 +1,71 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\DirectMessage;
+use App\Models\DirectThread;
+use Illuminate\Console\Attributes\Description;
+use Illuminate\Console\Attributes\Signature;
+use Illuminate\Console\Command;
+
+#[Signature('direct:scrape-messages')]
+#[Description('Scrape Instagram Direct threads and messages via Playwright')]
+class ScrapeDirectMessages extends RunsPlaywrightScript
+{
+    protected string $script = 'scrape-messages.spec.js';
+    protected int $processTimeout = 600;
+
+    protected function afterRun(): void
+    {
+        $path = base_path('playwright/output/scrape-messages.json');
+
+        if (! file_exists($path)) {
+            $this->warn('No output file found — nothing to upsert.');
+            return;
+        }
+
+        $data = json_decode(file_get_contents($path), true);
+
+        foreach ($data['threads'] ?? [] as $thread) {
+            DirectThread::updateOrCreate(
+                ['thread_id' => $thread['thread_id']],
+                [
+                    'thread_key' => $thread['thread_key'],
+                    'thread_title' => $thread['thread_title'],
+                    'is_group' => $thread['is_group'],
+                    'participants' => $thread['participants'],
+                    'last_activity_at' => $thread['last_activity_at'],
+                    'muted' => $thread['muted'],
+                ]
+            );
+        }
+
+        foreach ($data['messages'] ?? [] as $message) {
+            DirectMessage::updateOrCreate(
+                ['id' => $message['id']],
+                [
+                    'thread_id' => $message['thread_id'],
+                    'user_id' => $message['user_id'],
+                    'username' => $message['username'],
+                    'full_name' => $message['full_name'],
+                    'item_type' => $message['content_type'],
+                    'text' => $message['text'],
+                    'timestamp' => $message['timestamp_ms'],
+                    'shared_url' => $message['share']['url'] ?? null,
+                    'shared_preview_image' => $message['share']['preview_image'] ?? null,
+                    'shared_title' => $message['share']['title'] ?? null,
+                    'shared_caption' => $message['share']['caption'] ?? null,
+                    'reactions' => $message['reactions'],
+                    'replied_to_id' => $message['replied_to_id'],
+                    'replied_to_text' => $message['replied_to_text'],
+                ]
+            );
+        }
+
+        $this->info(sprintf(
+            'Upserted %d threads and %d messages.',
+            count($data['threads'] ?? []),
+            count($data['messages'] ?? [])
+        ));
+    }
+}
@@ -0,0 +1,50 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Post;
+use Illuminate\Console\Attributes\Description;
+use Illuminate\Console\Attributes\Signature;
+
+#[Signature('direct:scrape-feed')]
+#[Description('Scrape the Instagram following feed via Playwright')]
+class ScrapeFeed extends RunsPlaywrightScript
+{
+    protected string $script = 'scrape-feed.spec.js';
+
+    protected function afterRun(): void
+    {
+        $path = base_path('playwright/output/scrape-feed.json');
+
+        if (! file_exists($path)) {
+            $this->warn('No output file found — nothing to upsert.');
+            return;
+        }
+
+        $posts = json_decode(file_get_contents($path), true);
+
+        foreach ($posts as $post) {
+            Post::updateOrCreate(
+                ['id' => $post['id']],
+                [
+                    'code' => $post['code'],
+                    'username' => $post['username'],
+                    'full_name' => $post['full_name'],
+                    'caption' => $post['caption'],
+                    'accessibility_caption' => $post['accessibility_caption'],
+                    'is_video' => $post['is_video'],
+                    'is_carousel' => $post['is_carousel'],
+                    'like_count' => $post['like_count'],
+                    'comment_count' => $post['comment_count'],
+                    'taken_at' => $post['taken_at'],
+                    'image_url' => $post['image_url'],
+                    'video_url' => $post['video_url'],
+                    'carousel_images' => $post['carousel_images'],
+                    'profile_pic_url' => $post['profile_pic_url'],
+                ]
+            );
+        }
+
+        $this->info('Upserted ' . count($posts) . ' posts.');
+    }
+}
@@ -0,0 +1,47 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Story;
+use Illuminate\Console\Attributes\Description;
+use Illuminate\Console\Attributes\Signature;
+
+#[Signature('direct:scrape-stories')]
+#[Description('Scrape Instagram story content via Playwright')]
+class ScrapeStories extends RunsPlaywrightScript
+{
+    protected string $script = 'scrape-story-content.spec.js';
+
+    protected function afterRun(): void
+    {
+        $path = base_path('playwright/output/scrape-stories.json');
+
+        if (! file_exists($path)) {
+            $this->warn('No output file found — nothing to upsert.');
+            return;
+        }
+
+        $items = json_decode(file_get_contents($path), true);
+
+        foreach ($items as $item) {
+            Story::updateOrCreate(
+                ['id' => $item['id']],
+                [
+                    'user_id' => $item['user_id'],
+                    'username' => $item['username'],
+                    'profile_pic_url' => $item['profile_pic_url'],
+                    'code' => $item['code'],
+                    'is_video' => $item['is_video'],
+                    'taken_at' => $item['taken_at'],
+                    'expiring_at' => $item['expiring_at'],
+                    'image_url' => $item['image_url'],
+                    'video_url' => $item['video_url'],
+                    'accessibility_caption' => $item['accessibility_caption'],
+                    'muted' => $item['muted'],
+                ]
+            );
+        }
+
+        $this->info('Upserted ' . count($items) . ' story items.');
+    }
+}
@@ -0,0 +1,23 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    public function up(): void
+    {
+        Schema::create('settings', function (Blueprint $table) {
+            $table->id();
+            $table->string('key')->unique();
+            $table->text('value')->nullable();
+            $table->timestamps();
+        });
+    }
+
+    public function down(): void
+    {
+        Schema::dropIfExists('settings');
+    }
+};
@@ -0,0 +1,41 @@
+import { test } from '@playwright/test';
+
+test.use({ storageState: '/home/pwuser/ig-session.json' });
+
+test('inspect non-text message shapes', async ({ page }) => {
+    test.setTimeout(120_000);
+
+    const interesting = [];
+
+    page.on('response', async (response) => {
+        const url = response.url();
+        if (!url.includes('/api/graphql')) return;
+        const params = new URLSearchParams(response.request().postData() || '');
+        if (params.get('fb_api_req_friendly_name') !== 'IGDThreadDetailQuery') return;
+        try {
+            const json = await response.json();
+            const t = json?.data?.get_slide_thread_nullable?.as_ig_direct_thread;
+            if (!t) return;
+            for (const edge of t.slide_messages?.edges || []) {
+                const node = edge.node;
+                const hasReactions = (node.reactions?.length || 0) > 0 || (node.msg_reactions?.length || 0) > 0;
+                if (node.content_type !== 'TEXT' || hasReactions || node.replied_to_message) {
+                    interesting.push({ thread: t.thread_title, node });
+                }
+            }
+        } catch (e) { /* skip */ }
+    });
+
+    await page.goto('https://www.instagram.com/direct/inbox/');
+    await page.waitForTimeout(6000);
+
+    const threadHrefs = await page.$$eval('a[href*="/direct/t/"]', els =>
+        [...new Set(els.map(el => el.getAttribute('href')))]);
+
+    for (const href of threadHrefs) {
+        await page.goto(`https://www.instagram.com${href}`);
+        await page.waitForTimeout(3500);
+    }
+
+    console.log('[INTERESTING MESSAGES]', JSON.stringify(interesting, null, 2));
+});
@@ -0,0 +1,10 @@
+import fs from 'fs';
+import path from 'path';
+import { getStorageState } from './pwhelpers.js';
+
+export default async function globalSetup() {
+    const storageState = await getStorageState();
+    const outPath = path.resolve(__dirname, '.auth', 'ig-session.json');
+    fs.mkdirSync(path.dirname(outPath), { recursive: true });
+    fs.writeFileSync(outPath, JSON.stringify(storageState));
+}
@@ -0,0 +1,236 @@
+{
+  "name": "pwuser",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "pwuser",
+      "version": "1.0.0",
+      "license": "ISC",
+      "dependencies": {
+        "@playwright/test": "1.62.1",
+        "dotenv": "^17.4.2",
+        "pg": "^8.22.0",
+        "playwright": "1.62.1"
+      }
+    },
+    "node_modules/@playwright/test": {
+      "version": "1.62.1",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.1.tgz",
+      "integrity": "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright": "1.62.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=20"
+      }
+    },
+    "node_modules/dotenv": {
+      "version": "17.4.2",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz",
+      "integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==",
+      "license": "BSD-2-Clause",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://dotenvx.com"
+      }
+    },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
+    "node_modules/pg": {
+      "version": "8.22.0",
+      "resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz",
+      "integrity": "sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==",
+      "license": "MIT",
+      "dependencies": {
+        "pg-connection-string": "^2.14.0",
+        "pg-pool": "^3.14.0",
+        "pg-protocol": "^1.15.0",
+        "pg-types": "2.2.0",
+        "pgpass": "1.0.5"
+      },
+      "engines": {
+        "node": ">= 16.0.0"
+      },
+      "optionalDependencies": {
+        "pg-cloudflare": "^1.4.0"
+      },
+      "peerDependencies": {
+        "pg-native": ">=3.0.1"
+      },
+      "peerDependenciesMeta": {
+        "pg-native": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/pg-cloudflare": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz",
+      "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==",
+      "license": "MIT",
+      "optional": true
+    },
+    "node_modules/pg-connection-string": {
+      "version": "2.14.0",
+      "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz",
+      "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==",
+      "license": "MIT"
+    },
+    "node_modules/pg-int8": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
+      "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
+      "license": "ISC",
+      "engines": {
+        "node": ">=4.0.0"
+      }
+    },
+    "node_modules/pg-pool": {
+      "version": "3.14.0",
+      "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz",
+      "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==",
+      "license": "MIT",
+      "peerDependencies": {
+        "pg": ">=8.0"
+      }
+    },
+    "node_modules/pg-protocol": {
+      "version": "1.15.0",
+      "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.15.0.tgz",
+      "integrity": "sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==",
+      "license": "MIT"
+    },
+    "node_modules/pg-types": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
+      "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
+      "license": "MIT",
+      "dependencies": {
+        "pg-int8": "1.0.1",
+        "postgres-array": "~2.0.0",
+        "postgres-bytea": "~1.0.0",
+        "postgres-date": "~1.0.4",
+        "postgres-interval": "^1.1.0"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/pgpass": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
+      "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
+      "license": "MIT",
+      "dependencies": {
+        "split2": "^4.1.0"
+      }
+    },
+    "node_modules/playwright": {
+      "version": "1.62.1",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz",
+      "integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright-core": "1.62.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=20"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.62.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz",
+      "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==",
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=20"
+      }
+    },
+    "node_modules/postgres-array": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
+      "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/postgres-bytea": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz",
+      "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/postgres-date": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
+      "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/postgres-interval": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
+      "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
+      "license": "MIT",
+      "dependencies": {
+        "xtend": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/split2": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
+      "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
+      "license": "ISC",
+      "engines": {
+        "node": ">= 10.x"
+      }
+    },
+    "node_modules/xtend": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
+      "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.4"
+      }
+    }
+  }
+}
@@ -0,0 +1,19 @@
+{
+  "name": "pwuser",
+  "version": "1.0.0",
+  "description": "",
+  "main": "instagram-test.spec.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "type": "commonjs",
+  "dependencies": {
+    "@playwright/test": "1.62.1",
+    "dotenv": "^17.4.2",
+    "pg": "^8.22.0",
+    "playwright": "1.62.1"
+  }
+}
@@ -0,0 +1,12 @@
+import { defineConfig } from '@playwright/test';
+import path from 'path';
+
+export default defineConfig({
+    testDir: '.',
+    globalSetup: path.resolve(__dirname, 'global-setup.js'),
+    timeout: 120_000,
+    reporter: 'line',
+    use: {
+        headless: true,
+    },
+});
@@ -0,0 +1,31 @@
+import dotenv from 'dotenv';
+import path from 'path';
+import pg from 'pg';
+
+dotenv.config({ path: path.resolve(__dirname, '../.env') });
+
+export function getDbClient() {
+    return new pg.Client({
+        host: process.env.DB_HOST,
+        port: Number(process.env.DB_PORT) || 5432,
+        user: process.env.DB_USERNAME,
+        password: process.env.DB_PASSWORD,
+        database: process.env.DB_DATABASE,
+    });
+}
+
+export async function getStorageState() {
+    const client = getDbClient();
+    await client.connect();
+    try {
+        const { rows } = await client.query(
+            `SELECT value FROM settings WHERE key = $1 LIMIT 1`,
+            ['session']
+        );
+        if (!rows.length) throw new Error("No 'session' row found in settings table");
+        const { value } = rows[0];
+        return typeof value === 'string' ? JSON.parse(value) : value;
+    } finally {
+        await client.end();
+    }
+}
@@ -0,0 +1,106 @@
+import { test } from '@playwright/test';
+import fs from 'fs';
+import path from 'path';
+
+test.use({ storageState: path.resolve(__dirname, '.auth', 'ig-session.json') });
+function randomWait(min = 2500, max = 5000) {
+    return Math.floor(Math.random() * (max - min + 1)) + min;
+}
+
+function extractPostsFromMedia(m) {
+    return {
+        id: m.pk,
+        code: m.code,
+        username: m.user?.username,
+        full_name: m.user?.full_name,
+        profile_pic_url:
+            m.user?.hd_profile_pic_url_info?.url ||
+            m.user?.profile_pic_url ||
+            null,
+        caption: m.caption?.text || null,
+        accessibility_caption: m.accessibility_caption || null,
+        is_video: m.media_type === 2,
+        is_carousel: m.media_type === 8,
+        like_count: m.like_count,
+        comment_count: m.comment_count,
+        taken_at: m.taken_at,
+        image_url: m.image_versions2?.candidates?.[0]?.url || null,
+        video_url: m.video_versions?.[0]?.url || null,
+        carousel_images:
+            m.carousel_media?.map((item) => ({
+                url: item.image_versions2?.candidates?.[0]?.url || null,
+                video_url: item.video_versions?.[0]?.url || null,
+                accessibility_caption: item.accessibility_caption || null,
+            })) || null,
+    };
+}
+
+test('scrape following feed', async ({ page }) => {
+    test.setTimeout(120_000);
+    const posts = [];
+
+    page.on('response', async (response) => {
+        const url = response.url();
+        if (!url.includes('/graphql/query')) return;
+        try {
+            const json = await response.json();
+            const edges =
+                json?.data?.xdt_api__v1__feed__timeline__connection?.edges;
+            if (!edges) return;
+            for (const edge of edges) {
+                const m = edge.node?.media;
+                if (m) posts.push(extractPostsFromMedia(m));
+            }
+        } catch (e) {
+            /* not JSON, ignore */
+        }
+    });
+
+    await page.goto('https://www.instagram.com/?variant=following&hl=en');
+    await page.waitForTimeout(4000);
+
+    const preloadedEdges = await page.evaluate(() => {
+        function deepFind(obj, targetKey) {
+            if (obj === null || typeof obj !== 'object') return null;
+            if (targetKey in obj) return obj[targetKey];
+            for (const value of Object.values(obj)) {
+                const found = deepFind(value, targetKey);
+                if (found) return found;
+            }
+            return null;
+        }
+
+        const results = [];
+        document
+            .querySelectorAll('script[type="application/json"]')
+            .forEach((script) => {
+                try {
+                    const json = JSON.parse(script.textContent);
+                    const connection = deepFind(
+                        json,
+                        'xdt_api__v1__feed__timeline__connection',
+                    );
+                    if (connection?.edges) {
+                        results.push(...connection.edges);
+                    }
+                } catch (e) {
+                    /* not parseable JSON, skip */
+                }
+            });
+        return results;
+    });
+
+    for (const edge of preloadedEdges) {
+        const m = edge.node?.media;
+        if (m) posts.push(extractPostsFromMedia(m));
+    }
+
+    for (let i = 0; i < 10; i++) {
+        await page.mouse.wheel(0, 3000);
+        await page.waitForTimeout(randomWait());
+    }
+
+    const outputPath = path.resolve(__dirname, 'output', 'scrape-feed.json');
+    fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+    fs.writeFileSync(outputPath, JSON.stringify(posts));
+});
@@ -0,0 +1,171 @@
+import { test } from '@playwright/test';
+import fs from 'fs';
+import path from 'path';
+
+test.use({ storageState: path.resolve(__dirname, '.auth', 'ig-session.json') });
+
+function randomWait(min = 2500, max = 5000) {
+    return Math.floor(Math.random() * (max - min + 1)) + min;
+}
+
+function extractThread(t) {
+    const isGroup =
+        (t.thread_subtype || '').includes('GROUP') ||
+        (t.users || []).length > 1;
+    return {
+        thread_id: t.id,
+        thread_key: t.thread_key,
+        thread_title: t.thread_title || null,
+        is_group: isGroup,
+        participants: (t.users || []).map((u) => ({
+            user_id: u.id,
+            username: u.username,
+            full_name: u.full_name,
+            profile_pic_url: u.profile_pic_url || null,
+            interop_messaging_user_fbid: u.interop_messaging_user_fbid,
+        })),
+        last_activity_at: t.last_activity_timestamp_ms
+            ? Number(t.last_activity_timestamp_ms)
+            : null,
+        muted: t.is_muted ?? null,
+    };
+}
+
+// Inbox-preview messages only carry sender_fbid; thread-detail messages carry
+// a full sender.user_dict. Build a lookup so preview messages, and reactions
+// (which are keyed by sender_fbid too), still resolve to a username.
+function buildFbidLookup(t) {
+    const lookup = {};
+    for (const u of t.users || []) {
+        lookup[u.interop_messaging_user_fbid] = {
+            user_id: u.id,
+            username: u.username,
+            full_name: u.full_name,
+        };
+    }
+    return lookup;
+}
+
+function extractShare(node) {
+    const xma = node.content?.xma;
+    if (!xma) return null;
+    return {
+        url: xma.target_url || null,
+        preview_image: xma.preview_image?.url || xma.header_icon?.url || null,
+        title: xma.header_title_text || xma.title_text || null,
+        caption:
+            xma.caption_body_text ||
+            xma.eyebrow_text ||
+            xma.header_subtitle_text ||
+            null,
+    };
+}
+
+function extractMessage(node, thread, fbidLookup) {
+    const senderDict = node.sender?.user_dict;
+    const resolved = senderDict
+        ? {
+              user_id: node.sender.igid,
+              username: senderDict.username,
+              full_name: senderDict.full_name,
+          }
+        : fbidLookup[node.sender_fbid] || {};
+    const reactions = (node.reactions || []).map((r) => ({
+        emoji: r.reaction,
+        sender_fbid: r.sender_fbid,
+        username: fbidLookup[r.sender_fbid]?.username || null,
+    }));
+    return {
+        id: node.id || node.message_id,
+        thread_id: thread.id,
+        user_id: resolved.user_id || null,
+        username: resolved.username || null,
+        full_name: resolved.full_name || null,
+        content_type: node.content_type,
+        text: node.text_body || node.content?.text_body || null,
+        timestamp_ms: node.timestamp_ms ? Number(node.timestamp_ms) : null,
+        share: extractShare(node),
+        reactions: reactions.length ? reactions : null,
+        replied_to_id: node.replied_to_message_id || null,
+        replied_to_text: node.replied_to_message?.text_body || null,
+        hasDetail: !!senderDict,
+    };
+}
+
+test('fetch all direct message content', async ({ page }) => {
+    test.setTimeout(180_000);
+
+    const threadsMap = new Map();
+    const messagesMap = new Map();
+
+    function ingestThread(t) {
+        if (!t?.id) return;
+        threadsMap.set(t.id, extractThread(t));
+        const fbidLookup = buildFbidLookup(t);
+        for (const edge of t.slide_messages?.edges || []) {
+            const msg = extractMessage(edge.node, t, fbidLookup);
+            const existing = messagesMap.get(msg.id);
+            if (existing && existing.hasDetail && !msg.hasDetail) continue; // don't clobber a richer entry
+            messagesMap.set(msg.id, msg);
+        }
+    }
+
+    page.on('response', async (response) => {
+        const url = response.url();
+        if (!url.includes('/api/graphql')) return;
+        const params = new URLSearchParams(response.request().postData() || '');
+        const friendlyName = params.get('fb_api_req_friendly_name');
+        if (
+            friendlyName !== 'PolarisDirectInboxQuery' &&
+            friendlyName !== 'IGDThreadDetailQuery'
+        )
+            return;
+        try {
+            const json = await response.json();
+            if (friendlyName === 'PolarisDirectInboxQuery') {
+                const edges =
+                    json?.data?.get_slide_mailbox_for_iris_subscription
+                        ?.threads_by_folder?.edges || [];
+                for (const edge of edges)
+                    ingestThread(edge.node?.as_ig_direct_thread);
+            } else {
+                ingestThread(
+                    json?.data?.get_slide_thread_nullable?.as_ig_direct_thread,
+                );
+            }
+        } catch (e) {
+            /* not JSON, ignore */
+        }
+    });
+
+    await page.goto('https://www.instagram.com/direct/inbox/');
+    await page.waitForTimeout(6000);
+
+    // Open each conversation so its fuller message history loads via
+    // IGDThreadDetailQuery, using the real hrefs Instagram rendered rather
+    // than guessing the thread-id format ourselves
+    const threadHrefs = await page.$$eval('a[href*="/direct/t/"]', (els) => [
+        ...new Set(els.map((el) => el.getAttribute('href'))),
+    ]);
+
+    for (const href of threadHrefs) {
+        await page.goto(`https://www.instagram.com${href}`);
+        await page.waitForTimeout(randomWait());
+    }
+
+    const outputPath = path.resolve(
+        __dirname,
+        'output',
+        'scrape-messages.json',
+    );
+    fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+    fs.writeFileSync(
+        outputPath,
+        JSON.stringify({
+            threads: [...threadsMap.values()],
+            messages: [...messagesMap.values()].map(
+                ({ hasDetail, ...msg }) => msg,
+            ),
+        }),
+    );
+});
@@ -0,0 +1,134 @@
+import { test } from '@playwright/test';
+import fs from 'fs';
+import path from 'path';
+
+test.use({ storageState: path.resolve(__dirname, '.auth', 'ig-session.json') });
+
+function deepFind(obj, targetKey) {
+    if (obj === null || typeof obj !== 'object') return null;
+    if (targetKey in obj) return obj[targetKey];
+    for (const value of Object.values(obj)) {
+        const found = deepFind(value, targetKey);
+        if (found) return found;
+    }
+    return null;
+}
+
+function extractStoryContent(node) {
+    return {
+        user_id: node.id,
+        username: node.user?.username,
+        profile_pic_url: node.user?.profile_pic_url || null,
+        seen: node.seen,
+        latest_reel_media: node.latest_reel_media,
+        muted: node.muted,
+        items: (node.items || []).map((item) => ({
+            id: item.pk,
+            code: item.code,
+            is_video: item.media_type === 2,
+            taken_at: item.taken_at,
+            expiring_at: item.expiring_at,
+            image_url: item.image_versions2?.candidates?.[0]?.url || null,
+            video_url: item.video_versions?.[0]?.url || null,
+            accessibility_caption: item.accessibility_caption || null,
+        })),
+    };
+}
+
+test('fetch all story content directly', async ({ page }) => {
+    test.setTimeout(60_000);
+
+    // Load the page first so we have valid csrf/session context in cookies
+    await page.goto('https://www.instagram.com/');
+    await page.waitForTimeout(3000);
+
+    // Pull the tray's user IDs the same way as your existing stories scrape
+    const reelIds = await page.evaluate(() => {
+        function deepFind(obj, targetKey) {
+            if (obj === null || typeof obj !== 'object') return null;
+            if (targetKey in obj) return obj[targetKey];
+            for (const value of Object.values(obj)) {
+                const found = deepFind(value, targetKey);
+                if (found) return found;
+            }
+            return null;
+        }
+        const scripts = document.querySelectorAll(
+            'script[type="application/json"]',
+        );
+        for (const script of scripts) {
+            if (script.textContent.includes('xdt_api__v1__feed__reels_tray')) {
+                const json = JSON.parse(script.textContent);
+                const reelsTray = deepFind(
+                    json,
+                    'xdt_api__v1__feed__reels_tray',
+                );
+                return (reelsTray?.tray || []).map((item) => item.id);
+            }
+        }
+        return [];
+    });
+
+    if (reelIds.length === 0) {
+        console.log('No reel IDs found in tray - nothing to fetch');
+        return;
+    }
+
+    // Get the csrf token from cookies (needed as a header for the POST)
+    const cookies = await page.context().cookies();
+    const csrfToken = cookies.find((c) => c.name === 'csrftoken')?.value;
+
+    const variables = {
+        initial_reel_id: reelIds[0],
+        reel_ids: reelIds,
+        first: reelIds.length,
+        last: 0,
+        __relay_internal__pv__PolarisCommunityNoteStoriesLabelEnabledrelayprovider: true,
+        __relay_internal__pv__PolarisAIGMMediaWebLabelEnabledrelayprovider: false,
+    };
+
+    const body = new URLSearchParams({
+        fb_api_req_friendly_name: 'PolarisStoriesV3ReelPageGalleryQuery',
+        variables: JSON.stringify(variables),
+        doc_id: '27768924676024297',
+        server_timestamps: 'true',
+    });
+
+    const response = await page.request.post(
+        'https://www.instagram.com/graphql/query',
+        {
+            headers: {
+                'content-type': 'application/x-www-form-urlencoded',
+                'x-csrftoken': csrfToken,
+                'x-ig-app-id': '936619743392459',
+            },
+            data: body.toString(),
+        },
+    );
+
+    const json = await response.json();
+    const edges =
+        json?.data?.xdt_api__v1__feed__reels_media__connection?.edges || [];
+    const storyContent = edges.map((edge) => extractStoryContent(edge.node));
+
+    const storyItems = storyContent.flatMap((story) =>
+        story.items.map((item) => ({
+            id: item.id,
+            user_id: story.user_id,
+            username: story.username,
+            profile_pic_url: story.profile_pic_url,
+            code: item.code,
+            is_video: item.is_video,
+            taken_at: item.taken_at,
+            expiring_at: item.expiring_at,
+            image_url: item.image_url,
+            video_url: item.video_url,
+            accessibility_caption: item.accessibility_caption,
+            muted: story.muted,
+        })),
+    );
+
+    const outputPath = path.resolve(__dirname, 'output', 'scrape-stories.json');
+    fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+    fs.writeFileSync(outputPath, JSON.stringify(storyItems));
+});