Initial Commit
This commit is contained in:
@@ -27,3 +27,7 @@ yarn-error.log
|
|||||||
/.nova
|
/.nova
|
||||||
/.vscode
|
/.vscode
|
||||||
/.zed
|
/.zed
|
||||||
|
/playwright/node_modules
|
||||||
|
playwright/.auth/
|
||||||
|
playwright/output/
|
||||||
|
playwright/test-results
|
||||||
|
|||||||
@@ -0,0 +1,15 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App\Console\Commands;
|
||||||
|
|
||||||
|
use Illuminate\Console\Attributes\Description;
|
||||||
|
use Illuminate\Console\Attributes\Signature;
|
||||||
|
|
||||||
|
#[Signature('app:debug-playwright')]
|
||||||
|
#[Description('Command description')]
|
||||||
|
class DebugPlaywright extends RunsPlaywrightScript
|
||||||
|
{
|
||||||
|
protected $signature = 'direct:debug';
|
||||||
|
protected $description = 'Run the Playwright network/DOM debugging script against Instagram Direct inbox';
|
||||||
|
protected string $script = 'debugger.spec.js';
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App\Console\Commands;
|
||||||
|
|
||||||
|
use Illuminate\Console\Command;
|
||||||
|
use Illuminate\Support\Facades\Process;
|
||||||
|
|
||||||
|
abstract class RunsPlaywrightScript extends Command
|
||||||
|
{
|
||||||
|
protected string $script;
|
||||||
|
protected int $processTimeout = 300;
|
||||||
|
|
||||||
|
public function handle(): int
|
||||||
|
{
|
||||||
|
$this->info("Running {$this->script}...");
|
||||||
|
|
||||||
|
$result = Process::path(base_path('playwright'))
|
||||||
|
->timeout($this->processTimeout)
|
||||||
|
->run("npx playwright test {$this->script} --reporter=line");
|
||||||
|
|
||||||
|
$this->line($result->output());
|
||||||
|
|
||||||
|
if (! $result->successful()) {
|
||||||
|
$this->error($result->errorOutput());
|
||||||
|
return self::FAILURE;
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->afterRun();
|
||||||
|
|
||||||
|
$this->info('Done.');
|
||||||
|
return self::SUCCESS;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function afterRun(): void
|
||||||
|
{
|
||||||
|
// no-op by default
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App\Console\Commands;
|
||||||
|
|
||||||
|
use App\Models\DirectMessage;
|
||||||
|
use App\Models\DirectThread;
|
||||||
|
use Illuminate\Console\Attributes\Description;
|
||||||
|
use Illuminate\Console\Attributes\Signature;
|
||||||
|
use Illuminate\Console\Command;
|
||||||
|
|
||||||
|
#[Signature('direct:scrape-messages')]
|
||||||
|
#[Description('Scrape Instagram Direct threads and messages via Playwright')]
|
||||||
|
class ScrapeDirectMessages extends RunsPlaywrightScript
|
||||||
|
{
|
||||||
|
protected string $script = 'scrape-messages.spec.js';
|
||||||
|
protected int $processTimeout = 600;
|
||||||
|
|
||||||
|
protected function afterRun(): void
|
||||||
|
{
|
||||||
|
$path = base_path('playwright/output/scrape-messages.json');
|
||||||
|
|
||||||
|
if (! file_exists($path)) {
|
||||||
|
$this->warn('No output file found — nothing to upsert.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
$data = json_decode(file_get_contents($path), true);
|
||||||
|
|
||||||
|
foreach ($data['threads'] ?? [] as $thread) {
|
||||||
|
DirectThread::updateOrCreate(
|
||||||
|
['thread_id' => $thread['thread_id']],
|
||||||
|
[
|
||||||
|
'thread_key' => $thread['thread_key'],
|
||||||
|
'thread_title' => $thread['thread_title'],
|
||||||
|
'is_group' => $thread['is_group'],
|
||||||
|
'participants' => $thread['participants'],
|
||||||
|
'last_activity_at' => $thread['last_activity_at'],
|
||||||
|
'muted' => $thread['muted'],
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach ($data['messages'] ?? [] as $message) {
|
||||||
|
DirectMessage::updateOrCreate(
|
||||||
|
['id' => $message['id']],
|
||||||
|
[
|
||||||
|
'thread_id' => $message['thread_id'],
|
||||||
|
'user_id' => $message['user_id'],
|
||||||
|
'username' => $message['username'],
|
||||||
|
'full_name' => $message['full_name'],
|
||||||
|
'item_type' => $message['content_type'],
|
||||||
|
'text' => $message['text'],
|
||||||
|
'timestamp' => $message['timestamp_ms'],
|
||||||
|
'shared_url' => $message['share']['url'] ?? null,
|
||||||
|
'shared_preview_image' => $message['share']['preview_image'] ?? null,
|
||||||
|
'shared_title' => $message['share']['title'] ?? null,
|
||||||
|
'shared_caption' => $message['share']['caption'] ?? null,
|
||||||
|
'reactions' => $message['reactions'],
|
||||||
|
'replied_to_id' => $message['replied_to_id'],
|
||||||
|
'replied_to_text' => $message['replied_to_text'],
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->info(sprintf(
|
||||||
|
'Upserted %d threads and %d messages.',
|
||||||
|
count($data['threads'] ?? []),
|
||||||
|
count($data['messages'] ?? [])
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App\Console\Commands;
|
||||||
|
|
||||||
|
use App\Models\Post;
|
||||||
|
use Illuminate\Console\Attributes\Description;
|
||||||
|
use Illuminate\Console\Attributes\Signature;
|
||||||
|
|
||||||
|
#[Signature('direct:scrape-feed')]
|
||||||
|
#[Description('Scrape the Instagram following feed via Playwright')]
|
||||||
|
class ScrapeFeed extends RunsPlaywrightScript
|
||||||
|
{
|
||||||
|
protected string $script = 'scrape-feed.spec.js';
|
||||||
|
|
||||||
|
protected function afterRun(): void
|
||||||
|
{
|
||||||
|
$path = base_path('playwright/output/scrape-feed.json');
|
||||||
|
|
||||||
|
if (! file_exists($path)) {
|
||||||
|
$this->warn('No output file found — nothing to upsert.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
$posts = json_decode(file_get_contents($path), true);
|
||||||
|
|
||||||
|
foreach ($posts as $post) {
|
||||||
|
Post::updateOrCreate(
|
||||||
|
['id' => $post['id']],
|
||||||
|
[
|
||||||
|
'code' => $post['code'],
|
||||||
|
'username' => $post['username'],
|
||||||
|
'full_name' => $post['full_name'],
|
||||||
|
'caption' => $post['caption'],
|
||||||
|
'accessibility_caption' => $post['accessibility_caption'],
|
||||||
|
'is_video' => $post['is_video'],
|
||||||
|
'is_carousel' => $post['is_carousel'],
|
||||||
|
'like_count' => $post['like_count'],
|
||||||
|
'comment_count' => $post['comment_count'],
|
||||||
|
'taken_at' => $post['taken_at'],
|
||||||
|
'image_url' => $post['image_url'],
|
||||||
|
'video_url' => $post['video_url'],
|
||||||
|
'carousel_images' => $post['carousel_images'],
|
||||||
|
'profile_pic_url' => $post['profile_pic_url'],
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->info('Upserted ' . count($posts) . ' posts.');
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App\Console\Commands;
|
||||||
|
|
||||||
|
use App\Models\Story;
|
||||||
|
use Illuminate\Console\Attributes\Description;
|
||||||
|
use Illuminate\Console\Attributes\Signature;
|
||||||
|
|
||||||
|
#[Signature('direct:scrape-stories')]
|
||||||
|
#[Description('Scrape Instagram story content via Playwright')]
|
||||||
|
class ScrapeStories extends RunsPlaywrightScript
|
||||||
|
{
|
||||||
|
protected string $script = 'scrape-story-content.spec.js';
|
||||||
|
|
||||||
|
protected function afterRun(): void
|
||||||
|
{
|
||||||
|
$path = base_path('playwright/output/scrape-stories.json');
|
||||||
|
|
||||||
|
if (! file_exists($path)) {
|
||||||
|
$this->warn('No output file found — nothing to upsert.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
$items = json_decode(file_get_contents($path), true);
|
||||||
|
|
||||||
|
foreach ($items as $item) {
|
||||||
|
Story::updateOrCreate(
|
||||||
|
['id' => $item['id']],
|
||||||
|
[
|
||||||
|
'user_id' => $item['user_id'],
|
||||||
|
'username' => $item['username'],
|
||||||
|
'profile_pic_url' => $item['profile_pic_url'],
|
||||||
|
'code' => $item['code'],
|
||||||
|
'is_video' => $item['is_video'],
|
||||||
|
'taken_at' => $item['taken_at'],
|
||||||
|
'expiring_at' => $item['expiring_at'],
|
||||||
|
'image_url' => $item['image_url'],
|
||||||
|
'video_url' => $item['video_url'],
|
||||||
|
'accessibility_caption' => $item['accessibility_caption'],
|
||||||
|
'muted' => $item['muted'],
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->info('Upserted ' . count($items) . ' story items.');
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
use Illuminate\Database\Migrations\Migration;
|
||||||
|
use Illuminate\Database\Schema\Blueprint;
|
||||||
|
use Illuminate\Support\Facades\Schema;
|
||||||
|
|
||||||
|
return new class extends Migration
|
||||||
|
{
|
||||||
|
public function up(): void
|
||||||
|
{
|
||||||
|
Schema::create('settings', function (Blueprint $table) {
|
||||||
|
$table->id();
|
||||||
|
$table->string('key')->unique();
|
||||||
|
$table->text('value')->nullable();
|
||||||
|
$table->timestamps();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public function down(): void
|
||||||
|
{
|
||||||
|
Schema::dropIfExists('settings');
|
||||||
|
}
|
||||||
|
};
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
import { test } from '@playwright/test';
|
||||||
|
|
||||||
|
test.use({ storageState: '/home/pwuser/ig-session.json' });
|
||||||
|
|
||||||
|
test('inspect non-text message shapes', async ({ page }) => {
|
||||||
|
test.setTimeout(120_000);
|
||||||
|
|
||||||
|
const interesting = [];
|
||||||
|
|
||||||
|
page.on('response', async (response) => {
|
||||||
|
const url = response.url();
|
||||||
|
if (!url.includes('/api/graphql')) return;
|
||||||
|
const params = new URLSearchParams(response.request().postData() || '');
|
||||||
|
if (params.get('fb_api_req_friendly_name') !== 'IGDThreadDetailQuery') return;
|
||||||
|
try {
|
||||||
|
const json = await response.json();
|
||||||
|
const t = json?.data?.get_slide_thread_nullable?.as_ig_direct_thread;
|
||||||
|
if (!t) return;
|
||||||
|
for (const edge of t.slide_messages?.edges || []) {
|
||||||
|
const node = edge.node;
|
||||||
|
const hasReactions = (node.reactions?.length || 0) > 0 || (node.msg_reactions?.length || 0) > 0;
|
||||||
|
if (node.content_type !== 'TEXT' || hasReactions || node.replied_to_message) {
|
||||||
|
interesting.push({ thread: t.thread_title, node });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (e) { /* skip */ }
|
||||||
|
});
|
||||||
|
|
||||||
|
await page.goto('https://www.instagram.com/direct/inbox/');
|
||||||
|
await page.waitForTimeout(6000);
|
||||||
|
|
||||||
|
const threadHrefs = await page.$$eval('a[href*="/direct/t/"]', els =>
|
||||||
|
[...new Set(els.map(el => el.getAttribute('href')))]);
|
||||||
|
|
||||||
|
for (const href of threadHrefs) {
|
||||||
|
await page.goto(`https://www.instagram.com${href}`);
|
||||||
|
await page.waitForTimeout(3500);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('[INTERESTING MESSAGES]', JSON.stringify(interesting, null, 2));
|
||||||
|
});
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
import fs from 'fs';
|
||||||
|
import path from 'path';
|
||||||
|
import { getStorageState } from './pwhelpers.js';
|
||||||
|
|
||||||
|
export default async function globalSetup() {
|
||||||
|
const storageState = await getStorageState();
|
||||||
|
const outPath = path.resolve(__dirname, '.auth', 'ig-session.json');
|
||||||
|
fs.mkdirSync(path.dirname(outPath), { recursive: true });
|
||||||
|
fs.writeFileSync(outPath, JSON.stringify(storageState));
|
||||||
|
}
|
||||||
Generated
+236
@@ -0,0 +1,236 @@
|
|||||||
|
{
|
||||||
|
"name": "pwuser",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"lockfileVersion": 3,
|
||||||
|
"requires": true,
|
||||||
|
"packages": {
|
||||||
|
"": {
|
||||||
|
"name": "pwuser",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"license": "ISC",
|
||||||
|
"dependencies": {
|
||||||
|
"@playwright/test": "1.62.1",
|
||||||
|
"dotenv": "^17.4.2",
|
||||||
|
"pg": "^8.22.0",
|
||||||
|
"playwright": "1.62.1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@playwright/test": {
|
||||||
|
"version": "1.62.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.1.tgz",
|
||||||
|
"integrity": "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==",
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"dependencies": {
|
||||||
|
"playwright": "1.62.1"
|
||||||
|
},
|
||||||
|
"bin": {
|
||||||
|
"playwright": "cli.js"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/dotenv": {
|
||||||
|
"version": "17.4.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz",
|
||||||
|
"integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==",
|
||||||
|
"license": "BSD-2-Clause",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://dotenvx.com"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/fsevents": {
|
||||||
|
"version": "2.3.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
|
||||||
|
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
|
||||||
|
"hasInstallScript": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"darwin"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/pg": {
|
||||||
|
"version": "8.22.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz",
|
||||||
|
"integrity": "sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"pg-connection-string": "^2.14.0",
|
||||||
|
"pg-pool": "^3.14.0",
|
||||||
|
"pg-protocol": "^1.15.0",
|
||||||
|
"pg-types": "2.2.0",
|
||||||
|
"pgpass": "1.0.5"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 16.0.0"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"pg-cloudflare": "^1.4.0"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"pg-native": ">=3.0.1"
|
||||||
|
},
|
||||||
|
"peerDependenciesMeta": {
|
||||||
|
"pg-native": {
|
||||||
|
"optional": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/pg-cloudflare": {
|
||||||
|
"version": "1.4.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz",
|
||||||
|
"integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==",
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true
|
||||||
|
},
|
||||||
|
"node_modules/pg-connection-string": {
|
||||||
|
"version": "2.14.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz",
|
||||||
|
"integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==",
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
|
"node_modules/pg-int8": {
|
||||||
|
"version": "1.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
|
||||||
|
"integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
|
||||||
|
"license": "ISC",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=4.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/pg-pool": {
|
||||||
|
"version": "3.14.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz",
|
||||||
|
"integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==",
|
||||||
|
"license": "MIT",
|
||||||
|
"peerDependencies": {
|
||||||
|
"pg": ">=8.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/pg-protocol": {
|
||||||
|
"version": "1.15.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.15.0.tgz",
|
||||||
|
"integrity": "sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==",
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
|
"node_modules/pg-types": {
|
||||||
|
"version": "2.2.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
|
||||||
|
"integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"pg-int8": "1.0.1",
|
||||||
|
"postgres-array": "~2.0.0",
|
||||||
|
"postgres-bytea": "~1.0.0",
|
||||||
|
"postgres-date": "~1.0.4",
|
||||||
|
"postgres-interval": "^1.1.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=4"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/pgpass": {
|
||||||
|
"version": "1.0.5",
|
||||||
|
"resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
|
||||||
|
"integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"split2": "^4.1.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/playwright": {
|
||||||
|
"version": "1.62.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz",
|
||||||
|
"integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==",
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"dependencies": {
|
||||||
|
"playwright-core": "1.62.1"
|
||||||
|
},
|
||||||
|
"bin": {
|
||||||
|
"playwright": "cli.js"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"fsevents": "2.3.2"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/playwright-core": {
|
||||||
|
"version": "1.62.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz",
|
||||||
|
"integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==",
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"bin": {
|
||||||
|
"playwright-core": "cli.js"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/postgres-array": {
|
||||||
|
"version": "2.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
|
||||||
|
"integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=4"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/postgres-bytea": {
|
||||||
|
"version": "1.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz",
|
||||||
|
"integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==",
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=0.10.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/postgres-date": {
|
||||||
|
"version": "1.0.7",
|
||||||
|
"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
|
||||||
|
"integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=0.10.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/postgres-interval": {
|
||||||
|
"version": "1.2.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
|
||||||
|
"integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"xtend": "^4.0.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=0.10.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/split2": {
|
||||||
|
"version": "4.2.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
|
||||||
|
"integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
|
||||||
|
"license": "ISC",
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10.x"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/xtend": {
|
||||||
|
"version": "4.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
|
||||||
|
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=0.4"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
{
|
||||||
|
"name": "pwuser",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"description": "",
|
||||||
|
"main": "instagram-test.spec.js",
|
||||||
|
"scripts": {
|
||||||
|
"test": "echo \"Error: no test specified\" && exit 1"
|
||||||
|
},
|
||||||
|
"keywords": [],
|
||||||
|
"author": "",
|
||||||
|
"license": "ISC",
|
||||||
|
"type": "commonjs",
|
||||||
|
"dependencies": {
|
||||||
|
"@playwright/test": "1.62.1",
|
||||||
|
"dotenv": "^17.4.2",
|
||||||
|
"pg": "^8.22.0",
|
||||||
|
"playwright": "1.62.1"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
import { defineConfig } from '@playwright/test';
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
|
export default defineConfig({
|
||||||
|
testDir: '.',
|
||||||
|
globalSetup: path.resolve(__dirname, 'global-setup.js'),
|
||||||
|
timeout: 120_000,
|
||||||
|
reporter: 'line',
|
||||||
|
use: {
|
||||||
|
headless: true,
|
||||||
|
},
|
||||||
|
});
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
import dotenv from 'dotenv';
|
||||||
|
import path from 'path';
|
||||||
|
import pg from 'pg';
|
||||||
|
|
||||||
|
dotenv.config({ path: path.resolve(__dirname, '../.env') });
|
||||||
|
|
||||||
|
export function getDbClient() {
|
||||||
|
return new pg.Client({
|
||||||
|
host: process.env.DB_HOST,
|
||||||
|
port: Number(process.env.DB_PORT) || 5432,
|
||||||
|
user: process.env.DB_USERNAME,
|
||||||
|
password: process.env.DB_PASSWORD,
|
||||||
|
database: process.env.DB_DATABASE,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getStorageState() {
|
||||||
|
const client = getDbClient();
|
||||||
|
await client.connect();
|
||||||
|
try {
|
||||||
|
const { rows } = await client.query(
|
||||||
|
`SELECT value FROM settings WHERE key = $1 LIMIT 1`,
|
||||||
|
['session']
|
||||||
|
);
|
||||||
|
if (!rows.length) throw new Error("No 'session' row found in settings table");
|
||||||
|
const { value } = rows[0];
|
||||||
|
return typeof value === 'string' ? JSON.parse(value) : value;
|
||||||
|
} finally {
|
||||||
|
await client.end();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
import { test } from '@playwright/test';
|
||||||
|
import fs from 'fs';
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
|
test.use({ storageState: path.resolve(__dirname, '.auth', 'ig-session.json') });
|
||||||
|
function randomWait(min = 2500, max = 5000) {
|
||||||
|
return Math.floor(Math.random() * (max - min + 1)) + min;
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractPostsFromMedia(m) {
|
||||||
|
return {
|
||||||
|
id: m.pk,
|
||||||
|
code: m.code,
|
||||||
|
username: m.user?.username,
|
||||||
|
full_name: m.user?.full_name,
|
||||||
|
profile_pic_url:
|
||||||
|
m.user?.hd_profile_pic_url_info?.url ||
|
||||||
|
m.user?.profile_pic_url ||
|
||||||
|
null,
|
||||||
|
caption: m.caption?.text || null,
|
||||||
|
accessibility_caption: m.accessibility_caption || null,
|
||||||
|
is_video: m.media_type === 2,
|
||||||
|
is_carousel: m.media_type === 8,
|
||||||
|
like_count: m.like_count,
|
||||||
|
comment_count: m.comment_count,
|
||||||
|
taken_at: m.taken_at,
|
||||||
|
image_url: m.image_versions2?.candidates?.[0]?.url || null,
|
||||||
|
video_url: m.video_versions?.[0]?.url || null,
|
||||||
|
carousel_images:
|
||||||
|
m.carousel_media?.map((item) => ({
|
||||||
|
url: item.image_versions2?.candidates?.[0]?.url || null,
|
||||||
|
video_url: item.video_versions?.[0]?.url || null,
|
||||||
|
accessibility_caption: item.accessibility_caption || null,
|
||||||
|
})) || null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
test('scrape following feed', async ({ page }) => {
|
||||||
|
test.setTimeout(120_000);
|
||||||
|
const posts = [];
|
||||||
|
|
||||||
|
page.on('response', async (response) => {
|
||||||
|
const url = response.url();
|
||||||
|
if (!url.includes('/graphql/query')) return;
|
||||||
|
try {
|
||||||
|
const json = await response.json();
|
||||||
|
const edges =
|
||||||
|
json?.data?.xdt_api__v1__feed__timeline__connection?.edges;
|
||||||
|
if (!edges) return;
|
||||||
|
for (const edge of edges) {
|
||||||
|
const m = edge.node?.media;
|
||||||
|
if (m) posts.push(extractPostsFromMedia(m));
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
/* not JSON, ignore */
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
await page.goto('https://www.instagram.com/?variant=following&hl=en');
|
||||||
|
await page.waitForTimeout(4000);
|
||||||
|
|
||||||
|
const preloadedEdges = await page.evaluate(() => {
|
||||||
|
function deepFind(obj, targetKey) {
|
||||||
|
if (obj === null || typeof obj !== 'object') return null;
|
||||||
|
if (targetKey in obj) return obj[targetKey];
|
||||||
|
for (const value of Object.values(obj)) {
|
||||||
|
const found = deepFind(value, targetKey);
|
||||||
|
if (found) return found;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const results = [];
|
||||||
|
document
|
||||||
|
.querySelectorAll('script[type="application/json"]')
|
||||||
|
.forEach((script) => {
|
||||||
|
try {
|
||||||
|
const json = JSON.parse(script.textContent);
|
||||||
|
const connection = deepFind(
|
||||||
|
json,
|
||||||
|
'xdt_api__v1__feed__timeline__connection',
|
||||||
|
);
|
||||||
|
if (connection?.edges) {
|
||||||
|
results.push(...connection.edges);
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
/* not parseable JSON, skip */
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return results;
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const edge of preloadedEdges) {
|
||||||
|
const m = edge.node?.media;
|
||||||
|
if (m) posts.push(extractPostsFromMedia(m));
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let i = 0; i < 10; i++) {
|
||||||
|
await page.mouse.wheel(0, 3000);
|
||||||
|
await page.waitForTimeout(randomWait());
|
||||||
|
}
|
||||||
|
|
||||||
|
const outputPath = path.resolve(__dirname, 'output', 'scrape-feed.json');
|
||||||
|
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||||
|
fs.writeFileSync(outputPath, JSON.stringify(posts));
|
||||||
|
});
|
||||||
@@ -0,0 +1,171 @@
|
|||||||
|
import { test } from '@playwright/test';
|
||||||
|
import fs from 'fs';
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
|
test.use({ storageState: path.resolve(__dirname, '.auth', 'ig-session.json') });
|
||||||
|
|
||||||
|
function randomWait(min = 2500, max = 5000) {
|
||||||
|
return Math.floor(Math.random() * (max - min + 1)) + min;
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractThread(t) {
|
||||||
|
const isGroup =
|
||||||
|
(t.thread_subtype || '').includes('GROUP') ||
|
||||||
|
(t.users || []).length > 1;
|
||||||
|
return {
|
||||||
|
thread_id: t.id,
|
||||||
|
thread_key: t.thread_key,
|
||||||
|
thread_title: t.thread_title || null,
|
||||||
|
is_group: isGroup,
|
||||||
|
participants: (t.users || []).map((u) => ({
|
||||||
|
user_id: u.id,
|
||||||
|
username: u.username,
|
||||||
|
full_name: u.full_name,
|
||||||
|
profile_pic_url: u.profile_pic_url || null,
|
||||||
|
interop_messaging_user_fbid: u.interop_messaging_user_fbid,
|
||||||
|
})),
|
||||||
|
last_activity_at: t.last_activity_timestamp_ms
|
||||||
|
? Number(t.last_activity_timestamp_ms)
|
||||||
|
: null,
|
||||||
|
muted: t.is_muted ?? null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Inbox-preview messages only carry sender_fbid; thread-detail messages carry
|
||||||
|
// a full sender.user_dict. Build a lookup so preview messages, and reactions
|
||||||
|
// (which are keyed by sender_fbid too), still resolve to a username.
|
||||||
|
function buildFbidLookup(t) {
|
||||||
|
const lookup = {};
|
||||||
|
for (const u of t.users || []) {
|
||||||
|
lookup[u.interop_messaging_user_fbid] = {
|
||||||
|
user_id: u.id,
|
||||||
|
username: u.username,
|
||||||
|
full_name: u.full_name,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return lookup;
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractShare(node) {
|
||||||
|
const xma = node.content?.xma;
|
||||||
|
if (!xma) return null;
|
||||||
|
return {
|
||||||
|
url: xma.target_url || null,
|
||||||
|
preview_image: xma.preview_image?.url || xma.header_icon?.url || null,
|
||||||
|
title: xma.header_title_text || xma.title_text || null,
|
||||||
|
caption:
|
||||||
|
xma.caption_body_text ||
|
||||||
|
xma.eyebrow_text ||
|
||||||
|
xma.header_subtitle_text ||
|
||||||
|
null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractMessage(node, thread, fbidLookup) {
|
||||||
|
const senderDict = node.sender?.user_dict;
|
||||||
|
const resolved = senderDict
|
||||||
|
? {
|
||||||
|
user_id: node.sender.igid,
|
||||||
|
username: senderDict.username,
|
||||||
|
full_name: senderDict.full_name,
|
||||||
|
}
|
||||||
|
: fbidLookup[node.sender_fbid] || {};
|
||||||
|
const reactions = (node.reactions || []).map((r) => ({
|
||||||
|
emoji: r.reaction,
|
||||||
|
sender_fbid: r.sender_fbid,
|
||||||
|
username: fbidLookup[r.sender_fbid]?.username || null,
|
||||||
|
}));
|
||||||
|
return {
|
||||||
|
id: node.id || node.message_id,
|
||||||
|
thread_id: thread.id,
|
||||||
|
user_id: resolved.user_id || null,
|
||||||
|
username: resolved.username || null,
|
||||||
|
full_name: resolved.full_name || null,
|
||||||
|
content_type: node.content_type,
|
||||||
|
text: node.text_body || node.content?.text_body || null,
|
||||||
|
timestamp_ms: node.timestamp_ms ? Number(node.timestamp_ms) : null,
|
||||||
|
share: extractShare(node),
|
||||||
|
reactions: reactions.length ? reactions : null,
|
||||||
|
replied_to_id: node.replied_to_message_id || null,
|
||||||
|
replied_to_text: node.replied_to_message?.text_body || null,
|
||||||
|
hasDetail: !!senderDict,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
test('fetch all direct message content', async ({ page }) => {
|
||||||
|
test.setTimeout(180_000);
|
||||||
|
|
||||||
|
const threadsMap = new Map();
|
||||||
|
const messagesMap = new Map();
|
||||||
|
|
||||||
|
function ingestThread(t) {
|
||||||
|
if (!t?.id) return;
|
||||||
|
threadsMap.set(t.id, extractThread(t));
|
||||||
|
const fbidLookup = buildFbidLookup(t);
|
||||||
|
for (const edge of t.slide_messages?.edges || []) {
|
||||||
|
const msg = extractMessage(edge.node, t, fbidLookup);
|
||||||
|
const existing = messagesMap.get(msg.id);
|
||||||
|
if (existing && existing.hasDetail && !msg.hasDetail) continue; // don't clobber a richer entry
|
||||||
|
messagesMap.set(msg.id, msg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
page.on('response', async (response) => {
|
||||||
|
const url = response.url();
|
||||||
|
if (!url.includes('/api/graphql')) return;
|
||||||
|
const params = new URLSearchParams(response.request().postData() || '');
|
||||||
|
const friendlyName = params.get('fb_api_req_friendly_name');
|
||||||
|
if (
|
||||||
|
friendlyName !== 'PolarisDirectInboxQuery' &&
|
||||||
|
friendlyName !== 'IGDThreadDetailQuery'
|
||||||
|
)
|
||||||
|
return;
|
||||||
|
try {
|
||||||
|
const json = await response.json();
|
||||||
|
if (friendlyName === 'PolarisDirectInboxQuery') {
|
||||||
|
const edges =
|
||||||
|
json?.data?.get_slide_mailbox_for_iris_subscription
|
||||||
|
?.threads_by_folder?.edges || [];
|
||||||
|
for (const edge of edges)
|
||||||
|
ingestThread(edge.node?.as_ig_direct_thread);
|
||||||
|
} else {
|
||||||
|
ingestThread(
|
||||||
|
json?.data?.get_slide_thread_nullable?.as_ig_direct_thread,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
/* not JSON, ignore */
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
await page.goto('https://www.instagram.com/direct/inbox/');
|
||||||
|
await page.waitForTimeout(6000);
|
||||||
|
|
||||||
|
// Open each conversation so its fuller message history loads via
|
||||||
|
// IGDThreadDetailQuery, using the real hrefs Instagram rendered rather
|
||||||
|
// than guessing the thread-id format ourselves
|
||||||
|
const threadHrefs = await page.$$eval('a[href*="/direct/t/"]', (els) => [
|
||||||
|
...new Set(els.map((el) => el.getAttribute('href'))),
|
||||||
|
]);
|
||||||
|
|
||||||
|
for (const href of threadHrefs) {
|
||||||
|
await page.goto(`https://www.instagram.com${href}`);
|
||||||
|
await page.waitForTimeout(randomWait());
|
||||||
|
}
|
||||||
|
|
||||||
|
const outputPath = path.resolve(
|
||||||
|
__dirname,
|
||||||
|
'output',
|
||||||
|
'scrape-messages.json',
|
||||||
|
);
|
||||||
|
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||||
|
fs.writeFileSync(
|
||||||
|
outputPath,
|
||||||
|
JSON.stringify({
|
||||||
|
threads: [...threadsMap.values()],
|
||||||
|
messages: [...messagesMap.values()].map(
|
||||||
|
({ hasDetail, ...msg }) => msg,
|
||||||
|
),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
@@ -0,0 +1,134 @@
|
|||||||
|
import { test } from '@playwright/test';
|
||||||
|
import fs from 'fs';
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
|
test.use({ storageState: path.resolve(__dirname, '.auth', 'ig-session.json') });
|
||||||
|
|
||||||
|
function deepFind(obj, targetKey) {
|
||||||
|
if (obj === null || typeof obj !== 'object') return null;
|
||||||
|
if (targetKey in obj) return obj[targetKey];
|
||||||
|
for (const value of Object.values(obj)) {
|
||||||
|
const found = deepFind(value, targetKey);
|
||||||
|
if (found) return found;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractStoryContent(node) {
|
||||||
|
return {
|
||||||
|
user_id: node.id,
|
||||||
|
username: node.user?.username,
|
||||||
|
profile_pic_url: node.user?.profile_pic_url || null,
|
||||||
|
seen: node.seen,
|
||||||
|
latest_reel_media: node.latest_reel_media,
|
||||||
|
muted: node.muted,
|
||||||
|
items: (node.items || []).map((item) => ({
|
||||||
|
id: item.pk,
|
||||||
|
code: item.code,
|
||||||
|
is_video: item.media_type === 2,
|
||||||
|
taken_at: item.taken_at,
|
||||||
|
expiring_at: item.expiring_at,
|
||||||
|
image_url: item.image_versions2?.candidates?.[0]?.url || null,
|
||||||
|
video_url: item.video_versions?.[0]?.url || null,
|
||||||
|
accessibility_caption: item.accessibility_caption || null,
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
test('fetch all story content directly', async ({ page }) => {
|
||||||
|
test.setTimeout(60_000);
|
||||||
|
|
||||||
|
// Load the page first so we have valid csrf/session context in cookies
|
||||||
|
await page.goto('https://www.instagram.com/');
|
||||||
|
await page.waitForTimeout(3000);
|
||||||
|
|
||||||
|
// Pull the tray's user IDs the same way as your existing stories scrape
|
||||||
|
const reelIds = await page.evaluate(() => {
|
||||||
|
function deepFind(obj, targetKey) {
|
||||||
|
if (obj === null || typeof obj !== 'object') return null;
|
||||||
|
if (targetKey in obj) return obj[targetKey];
|
||||||
|
for (const value of Object.values(obj)) {
|
||||||
|
const found = deepFind(value, targetKey);
|
||||||
|
if (found) return found;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const scripts = document.querySelectorAll(
|
||||||
|
'script[type="application/json"]',
|
||||||
|
);
|
||||||
|
for (const script of scripts) {
|
||||||
|
if (script.textContent.includes('xdt_api__v1__feed__reels_tray')) {
|
||||||
|
const json = JSON.parse(script.textContent);
|
||||||
|
const reelsTray = deepFind(
|
||||||
|
json,
|
||||||
|
'xdt_api__v1__feed__reels_tray',
|
||||||
|
);
|
||||||
|
return (reelsTray?.tray || []).map((item) => item.id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return [];
|
||||||
|
});
|
||||||
|
|
||||||
|
if (reelIds.length === 0) {
|
||||||
|
console.log('No reel IDs found in tray - nothing to fetch');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the csrf token from cookies (needed as a header for the POST)
|
||||||
|
const cookies = await page.context().cookies();
|
||||||
|
const csrfToken = cookies.find((c) => c.name === 'csrftoken')?.value;
|
||||||
|
|
||||||
|
const variables = {
|
||||||
|
initial_reel_id: reelIds[0],
|
||||||
|
reel_ids: reelIds,
|
||||||
|
first: reelIds.length,
|
||||||
|
last: 0,
|
||||||
|
__relay_internal__pv__PolarisCommunityNoteStoriesLabelEnabledrelayprovider: true,
|
||||||
|
__relay_internal__pv__PolarisAIGMMediaWebLabelEnabledrelayprovider: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
const body = new URLSearchParams({
|
||||||
|
fb_api_req_friendly_name: 'PolarisStoriesV3ReelPageGalleryQuery',
|
||||||
|
variables: JSON.stringify(variables),
|
||||||
|
doc_id: '27768924676024297',
|
||||||
|
server_timestamps: 'true',
|
||||||
|
});
|
||||||
|
|
||||||
|
const response = await page.request.post(
|
||||||
|
'https://www.instagram.com/graphql/query',
|
||||||
|
{
|
||||||
|
headers: {
|
||||||
|
'content-type': 'application/x-www-form-urlencoded',
|
||||||
|
'x-csrftoken': csrfToken,
|
||||||
|
'x-ig-app-id': '936619743392459',
|
||||||
|
},
|
||||||
|
data: body.toString(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const json = await response.json();
|
||||||
|
const edges =
|
||||||
|
json?.data?.xdt_api__v1__feed__reels_media__connection?.edges || [];
|
||||||
|
const storyContent = edges.map((edge) => extractStoryContent(edge.node));
|
||||||
|
|
||||||
|
const storyItems = storyContent.flatMap((story) =>
|
||||||
|
story.items.map((item) => ({
|
||||||
|
id: item.id,
|
||||||
|
user_id: story.user_id,
|
||||||
|
username: story.username,
|
||||||
|
profile_pic_url: story.profile_pic_url,
|
||||||
|
code: item.code,
|
||||||
|
is_video: item.is_video,
|
||||||
|
taken_at: item.taken_at,
|
||||||
|
expiring_at: item.expiring_at,
|
||||||
|
image_url: item.image_url,
|
||||||
|
video_url: item.video_url,
|
||||||
|
accessibility_caption: item.accessibility_caption,
|
||||||
|
muted: story.muted,
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
|
||||||
|
const outputPath = path.resolve(__dirname, 'output', 'scrape-stories.json');
|
||||||
|
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||||
|
fs.writeFileSync(outputPath, JSON.stringify(storyItems));
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user