107 lines
3.6 KiB
JavaScript
107 lines
3.6 KiB
JavaScript
import { test } from '@playwright/test';
|
|
import fs from 'fs';
|
|
import path from 'path';
|
|
|
|
test.use({ storageState: path.resolve(__dirname, '.auth', 'ig-session.json') });
|
|
function randomWait(min = 2500, max = 5000) {
|
|
return Math.floor(Math.random() * (max - min + 1)) + min;
|
|
}
|
|
|
|
function extractPostsFromMedia(m) {
|
|
return {
|
|
id: m.pk,
|
|
code: m.code,
|
|
username: m.user?.username,
|
|
full_name: m.user?.full_name,
|
|
profile_pic_url:
|
|
m.user?.hd_profile_pic_url_info?.url ||
|
|
m.user?.profile_pic_url ||
|
|
null,
|
|
caption: m.caption?.text || null,
|
|
accessibility_caption: m.accessibility_caption || null,
|
|
is_video: m.media_type === 2,
|
|
is_carousel: m.media_type === 8,
|
|
like_count: m.like_count,
|
|
comment_count: m.comment_count,
|
|
taken_at: m.taken_at,
|
|
image_url: m.image_versions2?.candidates?.[0]?.url || null,
|
|
video_url: m.video_versions?.[0]?.url || null,
|
|
carousel_images:
|
|
m.carousel_media?.map((item) => ({
|
|
url: item.image_versions2?.candidates?.[0]?.url || null,
|
|
video_url: item.video_versions?.[0]?.url || null,
|
|
accessibility_caption: item.accessibility_caption || null,
|
|
})) || null,
|
|
};
|
|
}
|
|
|
|
test('scrape following feed', async ({ page }) => {
|
|
test.setTimeout(120_000);
|
|
const posts = [];
|
|
|
|
page.on('response', async (response) => {
|
|
const url = response.url();
|
|
if (!url.includes('/graphql/query')) return;
|
|
try {
|
|
const json = await response.json();
|
|
const edges =
|
|
json?.data?.xdt_api__v1__feed__timeline__connection?.edges;
|
|
if (!edges) return;
|
|
for (const edge of edges) {
|
|
const m = edge.node?.media;
|
|
if (m) posts.push(extractPostsFromMedia(m));
|
|
}
|
|
} catch (e) {
|
|
/* not JSON, ignore */
|
|
}
|
|
});
|
|
|
|
await page.goto('https://www.instagram.com/?variant=following&hl=en');
|
|
await page.waitForTimeout(4000);
|
|
|
|
const preloadedEdges = await page.evaluate(() => {
|
|
function deepFind(obj, targetKey) {
|
|
if (obj === null || typeof obj !== 'object') return null;
|
|
if (targetKey in obj) return obj[targetKey];
|
|
for (const value of Object.values(obj)) {
|
|
const found = deepFind(value, targetKey);
|
|
if (found) return found;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
const results = [];
|
|
document
|
|
.querySelectorAll('script[type="application/json"]')
|
|
.forEach((script) => {
|
|
try {
|
|
const json = JSON.parse(script.textContent);
|
|
const connection = deepFind(
|
|
json,
|
|
'xdt_api__v1__feed__timeline__connection',
|
|
);
|
|
if (connection?.edges) {
|
|
results.push(...connection.edges);
|
|
}
|
|
} catch (e) {
|
|
/* not parseable JSON, skip */
|
|
}
|
|
});
|
|
return results;
|
|
});
|
|
|
|
for (const edge of preloadedEdges) {
|
|
const m = edge.node?.media;
|
|
if (m) posts.push(extractPostsFromMedia(m));
|
|
}
|
|
|
|
for (let i = 0; i < 10; i++) {
|
|
await page.mouse.wheel(0, 3000);
|
|
await page.waitForTimeout(randomWait());
|
|
}
|
|
|
|
const outputPath = path.resolve(__dirname, 'output', 'scrape-feed.json');
|
|
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
|
fs.writeFileSync(outputPath, JSON.stringify(posts));
|
|
});
|