diff --git a/.gitignore b/.gitignore index 74e10e4..4b4f2f2 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,7 @@ yarn-error.log /.nova /.vscode /.zed +/playwright/node_modules +playwright/.auth/ +playwright/output/ +playwright/test-results diff --git a/app/Console/Commands/DebugPlaywright.php b/app/Console/Commands/DebugPlaywright.php new file mode 100644 index 0000000..393b240 --- /dev/null +++ b/app/Console/Commands/DebugPlaywright.php @@ -0,0 +1,15 @@ +info("Running {$this->script}..."); + + $result = Process::path(base_path('playwright')) + ->timeout($this->processTimeout) + ->run("npx playwright test {$this->script} --reporter=line"); + + $this->line($result->output()); + + if (! $result->successful()) { + $this->error($result->errorOutput()); + return self::FAILURE; + } + + $this->afterRun(); + + $this->info('Done.'); + return self::SUCCESS; + } + + protected function afterRun(): void + { + // no-op by default + } +} diff --git a/app/Console/Commands/ScrapeDirectMessages.php b/app/Console/Commands/ScrapeDirectMessages.php new file mode 100644 index 0000000..22a4885 --- /dev/null +++ b/app/Console/Commands/ScrapeDirectMessages.php @@ -0,0 +1,71 @@ +warn('No output file found — nothing to upsert.'); + return; + } + + $data = json_decode(file_get_contents($path), true); + + foreach ($data['threads'] ?? [] as $thread) { + DirectThread::updateOrCreate( + ['thread_id' => $thread['thread_id']], + [ + 'thread_key' => $thread['thread_key'], + 'thread_title' => $thread['thread_title'], + 'is_group' => $thread['is_group'], + 'participants' => $thread['participants'], + 'last_activity_at' => $thread['last_activity_at'], + 'muted' => $thread['muted'], + ] + ); + } + + foreach ($data['messages'] ?? [] as $message) { + DirectMessage::updateOrCreate( + ['id' => $message['id']], + [ + 'thread_id' => $message['thread_id'], + 'user_id' => $message['user_id'], + 'username' => $message['username'], + 'full_name' => $message['full_name'], + 'item_type' => $message['content_type'], + 'text' => $message['text'], + 'timestamp' => $message['timestamp_ms'], + 'shared_url' => $message['share']['url'] ?? null, + 'shared_preview_image' => $message['share']['preview_image'] ?? null, + 'shared_title' => $message['share']['title'] ?? null, + 'shared_caption' => $message['share']['caption'] ?? null, + 'reactions' => $message['reactions'], + 'replied_to_id' => $message['replied_to_id'], + 'replied_to_text' => $message['replied_to_text'], + ] + ); + } + + $this->info(sprintf( + 'Upserted %d threads and %d messages.', + count($data['threads'] ?? []), + count($data['messages'] ?? []) + )); + } +} diff --git a/app/Console/Commands/ScrapeFeed.php b/app/Console/Commands/ScrapeFeed.php new file mode 100644 index 0000000..9e7e422 --- /dev/null +++ b/app/Console/Commands/ScrapeFeed.php @@ -0,0 +1,50 @@ +warn('No output file found — nothing to upsert.'); + return; + } + + $posts = json_decode(file_get_contents($path), true); + + foreach ($posts as $post) { + Post::updateOrCreate( + ['id' => $post['id']], + [ + 'code' => $post['code'], + 'username' => $post['username'], + 'full_name' => $post['full_name'], + 'caption' => $post['caption'], + 'accessibility_caption' => $post['accessibility_caption'], + 'is_video' => $post['is_video'], + 'is_carousel' => $post['is_carousel'], + 'like_count' => $post['like_count'], + 'comment_count' => $post['comment_count'], + 'taken_at' => $post['taken_at'], + 'image_url' => $post['image_url'], + 'video_url' => $post['video_url'], + 'carousel_images' => $post['carousel_images'], + 'profile_pic_url' => $post['profile_pic_url'], + ] + ); + } + + $this->info('Upserted ' . count($posts) . ' posts.'); + } +} diff --git a/app/Console/Commands/ScrapeStories.php b/app/Console/Commands/ScrapeStories.php new file mode 100644 index 0000000..bff9c2a --- /dev/null +++ b/app/Console/Commands/ScrapeStories.php @@ -0,0 +1,47 @@ +warn('No output file found — nothing to upsert.'); + return; + } + + $items = json_decode(file_get_contents($path), true); + + foreach ($items as $item) { + Story::updateOrCreate( + ['id' => $item['id']], + [ + 'user_id' => $item['user_id'], + 'username' => $item['username'], + 'profile_pic_url' => $item['profile_pic_url'], + 'code' => $item['code'], + 'is_video' => $item['is_video'], + 'taken_at' => $item['taken_at'], + 'expiring_at' => $item['expiring_at'], + 'image_url' => $item['image_url'], + 'video_url' => $item['video_url'], + 'accessibility_caption' => $item['accessibility_caption'], + 'muted' => $item['muted'], + ] + ); + } + + $this->info('Upserted ' . count($items) . ' story items.'); + } +} diff --git a/database/migrations/2026_08_28_115603_create_settings_table.php b/database/migrations/2026_08_28_115603_create_settings_table.php new file mode 100644 index 0000000..5cc2756 --- /dev/null +++ b/database/migrations/2026_08_28_115603_create_settings_table.php @@ -0,0 +1,23 @@ +id(); + $table->string('key')->unique(); + $table->text('value')->nullable(); + $table->timestamps(); + }); + } + + public function down(): void + { + Schema::dropIfExists('settings'); + } +}; diff --git a/playwright/debugger.spec.js b/playwright/debugger.spec.js new file mode 100644 index 0000000..ca2ec83 --- /dev/null +++ b/playwright/debugger.spec.js @@ -0,0 +1,41 @@ +import { test } from '@playwright/test'; + +test.use({ storageState: '/home/pwuser/ig-session.json' }); + +test('inspect non-text message shapes', async ({ page }) => { + test.setTimeout(120_000); + + const interesting = []; + + page.on('response', async (response) => { + const url = response.url(); + if (!url.includes('/api/graphql')) return; + const params = new URLSearchParams(response.request().postData() || ''); + if (params.get('fb_api_req_friendly_name') !== 'IGDThreadDetailQuery') return; + try { + const json = await response.json(); + const t = json?.data?.get_slide_thread_nullable?.as_ig_direct_thread; + if (!t) return; + for (const edge of t.slide_messages?.edges || []) { + const node = edge.node; + const hasReactions = (node.reactions?.length || 0) > 0 || (node.msg_reactions?.length || 0) > 0; + if (node.content_type !== 'TEXT' || hasReactions || node.replied_to_message) { + interesting.push({ thread: t.thread_title, node }); + } + } + } catch (e) { /* skip */ } + }); + + await page.goto('https://www.instagram.com/direct/inbox/'); + await page.waitForTimeout(6000); + + const threadHrefs = await page.$$eval('a[href*="/direct/t/"]', els => + [...new Set(els.map(el => el.getAttribute('href')))]); + + for (const href of threadHrefs) { + await page.goto(`https://www.instagram.com${href}`); + await page.waitForTimeout(3500); + } + + console.log('[INTERESTING MESSAGES]', JSON.stringify(interesting, null, 2)); +}); \ No newline at end of file diff --git a/playwright/global-setup.js b/playwright/global-setup.js new file mode 100644 index 0000000..0ec7f6e --- /dev/null +++ b/playwright/global-setup.js @@ -0,0 +1,10 @@ +import fs from 'fs'; +import path from 'path'; +import { getStorageState } from './pwhelpers.js'; + +export default async function globalSetup() { + const storageState = await getStorageState(); + const outPath = path.resolve(__dirname, '.auth', 'ig-session.json'); + fs.mkdirSync(path.dirname(outPath), { recursive: true }); + fs.writeFileSync(outPath, JSON.stringify(storageState)); +} diff --git a/playwright/package-lock.json b/playwright/package-lock.json new file mode 100644 index 0000000..92a496d --- /dev/null +++ b/playwright/package-lock.json @@ -0,0 +1,236 @@ +{ + "name": "pwuser", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "pwuser", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@playwright/test": "1.62.1", + "dotenv": "^17.4.2", + "pg": "^8.22.0", + "playwright": "1.62.1" + } + }, + "node_modules/@playwright/test": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.1.tgz", + "integrity": "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==", + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/dotenv": { + "version": "17.4.2", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz", + "integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/pg": { + "version": "8.22.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz", + "integrity": "sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.0", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.15.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz", + "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.14.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz", + "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.15.0.tgz", + "integrity": "sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/playwright": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz", + "integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz", + "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + } + } +} diff --git a/playwright/package.json b/playwright/package.json new file mode 100644 index 0000000..fdcbe62 --- /dev/null +++ b/playwright/package.json @@ -0,0 +1,19 @@ +{ + "name": "pwuser", + "version": "1.0.0", + "description": "", + "main": "instagram-test.spec.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "type": "commonjs", + "dependencies": { + "@playwright/test": "1.62.1", + "dotenv": "^17.4.2", + "pg": "^8.22.0", + "playwright": "1.62.1" + } +} diff --git a/playwright/playwright.config.js b/playwright/playwright.config.js new file mode 100644 index 0000000..6c4169c --- /dev/null +++ b/playwright/playwright.config.js @@ -0,0 +1,12 @@ +import { defineConfig } from '@playwright/test'; +import path from 'path'; + +export default defineConfig({ + testDir: '.', + globalSetup: path.resolve(__dirname, 'global-setup.js'), + timeout: 120_000, + reporter: 'line', + use: { + headless: true, + }, +}); diff --git a/playwright/pwhelpers.js b/playwright/pwhelpers.js new file mode 100644 index 0000000..469b8b3 --- /dev/null +++ b/playwright/pwhelpers.js @@ -0,0 +1,31 @@ +import dotenv from 'dotenv'; +import path from 'path'; +import pg from 'pg'; + +dotenv.config({ path: path.resolve(__dirname, '../.env') }); + +export function getDbClient() { + return new pg.Client({ + host: process.env.DB_HOST, + port: Number(process.env.DB_PORT) || 5432, + user: process.env.DB_USERNAME, + password: process.env.DB_PASSWORD, + database: process.env.DB_DATABASE, + }); +} + +export async function getStorageState() { + const client = getDbClient(); + await client.connect(); + try { + const { rows } = await client.query( + `SELECT value FROM settings WHERE key = $1 LIMIT 1`, + ['session'] + ); + if (!rows.length) throw new Error("No 'session' row found in settings table"); + const { value } = rows[0]; + return typeof value === 'string' ? JSON.parse(value) : value; + } finally { + await client.end(); + } +} diff --git a/playwright/scrape-feed.spec.js b/playwright/scrape-feed.spec.js new file mode 100644 index 0000000..b3ad704 --- /dev/null +++ b/playwright/scrape-feed.spec.js @@ -0,0 +1,106 @@ +import { test } from '@playwright/test'; +import fs from 'fs'; +import path from 'path'; + +test.use({ storageState: path.resolve(__dirname, '.auth', 'ig-session.json') }); +function randomWait(min = 2500, max = 5000) { + return Math.floor(Math.random() * (max - min + 1)) + min; +} + +function extractPostsFromMedia(m) { + return { + id: m.pk, + code: m.code, + username: m.user?.username, + full_name: m.user?.full_name, + profile_pic_url: + m.user?.hd_profile_pic_url_info?.url || + m.user?.profile_pic_url || + null, + caption: m.caption?.text || null, + accessibility_caption: m.accessibility_caption || null, + is_video: m.media_type === 2, + is_carousel: m.media_type === 8, + like_count: m.like_count, + comment_count: m.comment_count, + taken_at: m.taken_at, + image_url: m.image_versions2?.candidates?.[0]?.url || null, + video_url: m.video_versions?.[0]?.url || null, + carousel_images: + m.carousel_media?.map((item) => ({ + url: item.image_versions2?.candidates?.[0]?.url || null, + video_url: item.video_versions?.[0]?.url || null, + accessibility_caption: item.accessibility_caption || null, + })) || null, + }; +} + +test('scrape following feed', async ({ page }) => { + test.setTimeout(120_000); + const posts = []; + + page.on('response', async (response) => { + const url = response.url(); + if (!url.includes('/graphql/query')) return; + try { + const json = await response.json(); + const edges = + json?.data?.xdt_api__v1__feed__timeline__connection?.edges; + if (!edges) return; + for (const edge of edges) { + const m = edge.node?.media; + if (m) posts.push(extractPostsFromMedia(m)); + } + } catch (e) { + /* not JSON, ignore */ + } + }); + + await page.goto('https://www.instagram.com/?variant=following&hl=en'); + await page.waitForTimeout(4000); + + const preloadedEdges = await page.evaluate(() => { + function deepFind(obj, targetKey) { + if (obj === null || typeof obj !== 'object') return null; + if (targetKey in obj) return obj[targetKey]; + for (const value of Object.values(obj)) { + const found = deepFind(value, targetKey); + if (found) return found; + } + return null; + } + + const results = []; + document + .querySelectorAll('script[type="application/json"]') + .forEach((script) => { + try { + const json = JSON.parse(script.textContent); + const connection = deepFind( + json, + 'xdt_api__v1__feed__timeline__connection', + ); + if (connection?.edges) { + results.push(...connection.edges); + } + } catch (e) { + /* not parseable JSON, skip */ + } + }); + return results; + }); + + for (const edge of preloadedEdges) { + const m = edge.node?.media; + if (m) posts.push(extractPostsFromMedia(m)); + } + + for (let i = 0; i < 10; i++) { + await page.mouse.wheel(0, 3000); + await page.waitForTimeout(randomWait()); + } + + const outputPath = path.resolve(__dirname, 'output', 'scrape-feed.json'); + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + fs.writeFileSync(outputPath, JSON.stringify(posts)); +}); diff --git a/playwright/scrape-messages.spec.js b/playwright/scrape-messages.spec.js new file mode 100644 index 0000000..c0749fd --- /dev/null +++ b/playwright/scrape-messages.spec.js @@ -0,0 +1,171 @@ +import { test } from '@playwright/test'; +import fs from 'fs'; +import path from 'path'; + +test.use({ storageState: path.resolve(__dirname, '.auth', 'ig-session.json') }); + +function randomWait(min = 2500, max = 5000) { + return Math.floor(Math.random() * (max - min + 1)) + min; +} + +function extractThread(t) { + const isGroup = + (t.thread_subtype || '').includes('GROUP') || + (t.users || []).length > 1; + return { + thread_id: t.id, + thread_key: t.thread_key, + thread_title: t.thread_title || null, + is_group: isGroup, + participants: (t.users || []).map((u) => ({ + user_id: u.id, + username: u.username, + full_name: u.full_name, + profile_pic_url: u.profile_pic_url || null, + interop_messaging_user_fbid: u.interop_messaging_user_fbid, + })), + last_activity_at: t.last_activity_timestamp_ms + ? Number(t.last_activity_timestamp_ms) + : null, + muted: t.is_muted ?? null, + }; +} + +// Inbox-preview messages only carry sender_fbid; thread-detail messages carry +// a full sender.user_dict. Build a lookup so preview messages, and reactions +// (which are keyed by sender_fbid too), still resolve to a username. +function buildFbidLookup(t) { + const lookup = {}; + for (const u of t.users || []) { + lookup[u.interop_messaging_user_fbid] = { + user_id: u.id, + username: u.username, + full_name: u.full_name, + }; + } + return lookup; +} + +function extractShare(node) { + const xma = node.content?.xma; + if (!xma) return null; + return { + url: xma.target_url || null, + preview_image: xma.preview_image?.url || xma.header_icon?.url || null, + title: xma.header_title_text || xma.title_text || null, + caption: + xma.caption_body_text || + xma.eyebrow_text || + xma.header_subtitle_text || + null, + }; +} + +function extractMessage(node, thread, fbidLookup) { + const senderDict = node.sender?.user_dict; + const resolved = senderDict + ? { + user_id: node.sender.igid, + username: senderDict.username, + full_name: senderDict.full_name, + } + : fbidLookup[node.sender_fbid] || {}; + const reactions = (node.reactions || []).map((r) => ({ + emoji: r.reaction, + sender_fbid: r.sender_fbid, + username: fbidLookup[r.sender_fbid]?.username || null, + })); + return { + id: node.id || node.message_id, + thread_id: thread.id, + user_id: resolved.user_id || null, + username: resolved.username || null, + full_name: resolved.full_name || null, + content_type: node.content_type, + text: node.text_body || node.content?.text_body || null, + timestamp_ms: node.timestamp_ms ? Number(node.timestamp_ms) : null, + share: extractShare(node), + reactions: reactions.length ? reactions : null, + replied_to_id: node.replied_to_message_id || null, + replied_to_text: node.replied_to_message?.text_body || null, + hasDetail: !!senderDict, + }; +} + +test('fetch all direct message content', async ({ page }) => { + test.setTimeout(180_000); + + const threadsMap = new Map(); + const messagesMap = new Map(); + + function ingestThread(t) { + if (!t?.id) return; + threadsMap.set(t.id, extractThread(t)); + const fbidLookup = buildFbidLookup(t); + for (const edge of t.slide_messages?.edges || []) { + const msg = extractMessage(edge.node, t, fbidLookup); + const existing = messagesMap.get(msg.id); + if (existing && existing.hasDetail && !msg.hasDetail) continue; // don't clobber a richer entry + messagesMap.set(msg.id, msg); + } + } + + page.on('response', async (response) => { + const url = response.url(); + if (!url.includes('/api/graphql')) return; + const params = new URLSearchParams(response.request().postData() || ''); + const friendlyName = params.get('fb_api_req_friendly_name'); + if ( + friendlyName !== 'PolarisDirectInboxQuery' && + friendlyName !== 'IGDThreadDetailQuery' + ) + return; + try { + const json = await response.json(); + if (friendlyName === 'PolarisDirectInboxQuery') { + const edges = + json?.data?.get_slide_mailbox_for_iris_subscription + ?.threads_by_folder?.edges || []; + for (const edge of edges) + ingestThread(edge.node?.as_ig_direct_thread); + } else { + ingestThread( + json?.data?.get_slide_thread_nullable?.as_ig_direct_thread, + ); + } + } catch (e) { + /* not JSON, ignore */ + } + }); + + await page.goto('https://www.instagram.com/direct/inbox/'); + await page.waitForTimeout(6000); + + // Open each conversation so its fuller message history loads via + // IGDThreadDetailQuery, using the real hrefs Instagram rendered rather + // than guessing the thread-id format ourselves + const threadHrefs = await page.$$eval('a[href*="/direct/t/"]', (els) => [ + ...new Set(els.map((el) => el.getAttribute('href'))), + ]); + + for (const href of threadHrefs) { + await page.goto(`https://www.instagram.com${href}`); + await page.waitForTimeout(randomWait()); + } + + const outputPath = path.resolve( + __dirname, + 'output', + 'scrape-messages.json', + ); + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + fs.writeFileSync( + outputPath, + JSON.stringify({ + threads: [...threadsMap.values()], + messages: [...messagesMap.values()].map( + ({ hasDetail, ...msg }) => msg, + ), + }), + ); +}); diff --git a/playwright/scrape-story-content.spec.js b/playwright/scrape-story-content.spec.js new file mode 100644 index 0000000..a1c1db1 --- /dev/null +++ b/playwright/scrape-story-content.spec.js @@ -0,0 +1,134 @@ +import { test } from '@playwright/test'; +import fs from 'fs'; +import path from 'path'; + +test.use({ storageState: path.resolve(__dirname, '.auth', 'ig-session.json') }); + +function deepFind(obj, targetKey) { + if (obj === null || typeof obj !== 'object') return null; + if (targetKey in obj) return obj[targetKey]; + for (const value of Object.values(obj)) { + const found = deepFind(value, targetKey); + if (found) return found; + } + return null; +} + +function extractStoryContent(node) { + return { + user_id: node.id, + username: node.user?.username, + profile_pic_url: node.user?.profile_pic_url || null, + seen: node.seen, + latest_reel_media: node.latest_reel_media, + muted: node.muted, + items: (node.items || []).map((item) => ({ + id: item.pk, + code: item.code, + is_video: item.media_type === 2, + taken_at: item.taken_at, + expiring_at: item.expiring_at, + image_url: item.image_versions2?.candidates?.[0]?.url || null, + video_url: item.video_versions?.[0]?.url || null, + accessibility_caption: item.accessibility_caption || null, + })), + }; +} + +test('fetch all story content directly', async ({ page }) => { + test.setTimeout(60_000); + + // Load the page first so we have valid csrf/session context in cookies + await page.goto('https://www.instagram.com/'); + await page.waitForTimeout(3000); + + // Pull the tray's user IDs the same way as your existing stories scrape + const reelIds = await page.evaluate(() => { + function deepFind(obj, targetKey) { + if (obj === null || typeof obj !== 'object') return null; + if (targetKey in obj) return obj[targetKey]; + for (const value of Object.values(obj)) { + const found = deepFind(value, targetKey); + if (found) return found; + } + return null; + } + const scripts = document.querySelectorAll( + 'script[type="application/json"]', + ); + for (const script of scripts) { + if (script.textContent.includes('xdt_api__v1__feed__reels_tray')) { + const json = JSON.parse(script.textContent); + const reelsTray = deepFind( + json, + 'xdt_api__v1__feed__reels_tray', + ); + return (reelsTray?.tray || []).map((item) => item.id); + } + } + return []; + }); + + if (reelIds.length === 0) { + console.log('No reel IDs found in tray - nothing to fetch'); + return; + } + + // Get the csrf token from cookies (needed as a header for the POST) + const cookies = await page.context().cookies(); + const csrfToken = cookies.find((c) => c.name === 'csrftoken')?.value; + + const variables = { + initial_reel_id: reelIds[0], + reel_ids: reelIds, + first: reelIds.length, + last: 0, + __relay_internal__pv__PolarisCommunityNoteStoriesLabelEnabledrelayprovider: true, + __relay_internal__pv__PolarisAIGMMediaWebLabelEnabledrelayprovider: false, + }; + + const body = new URLSearchParams({ + fb_api_req_friendly_name: 'PolarisStoriesV3ReelPageGalleryQuery', + variables: JSON.stringify(variables), + doc_id: '27768924676024297', + server_timestamps: 'true', + }); + + const response = await page.request.post( + 'https://www.instagram.com/graphql/query', + { + headers: { + 'content-type': 'application/x-www-form-urlencoded', + 'x-csrftoken': csrfToken, + 'x-ig-app-id': '936619743392459', + }, + data: body.toString(), + }, + ); + + const json = await response.json(); + const edges = + json?.data?.xdt_api__v1__feed__reels_media__connection?.edges || []; + const storyContent = edges.map((edge) => extractStoryContent(edge.node)); + + const storyItems = storyContent.flatMap((story) => + story.items.map((item) => ({ + id: item.id, + user_id: story.user_id, + username: story.username, + profile_pic_url: story.profile_pic_url, + code: item.code, + is_video: item.is_video, + taken_at: item.taken_at, + expiring_at: item.expiring_at, + image_url: item.image_url, + video_url: item.video_url, + accessibility_caption: item.accessibility_caption, + muted: story.muted, + })), + ); + + const outputPath = path.resolve(__dirname, 'output', 'scrape-stories.json'); + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + fs.writeFileSync(outputPath, JSON.stringify(storyItems)); +});