Initial commit

.gitignore

@@ -0,0 +1,3 @@
+scraped_liveries
+node_modules
+.idea

.idea/.gitignore

@@ -0,0 +1,10 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Ignored default folder with query files
+/queries/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

index.ts

@@ -0,0 +1,110 @@
+import axios from 'axios';
+import * as cheerio from 'cheerio';
+import * as fs from 'fs';
+import * as path from 'path';
+import https from 'https';
+import { pipeline } from 'stream/promises';
+
+https.globalAgent.setMaxListeners(50);
+
+const TARGET_URL = 'https://norebbostock.com/collections/all';
+const OUTPUT_DIR = './scraped_liveries';
+
+async function downloadImage(imageUrl: string, filename: string): Promise<void> {
+    const filepath = path.join(OUTPUT_DIR, filename);
+    const response = await axios.get(imageUrl, { responseType: 'stream' });
+    const writer = fs.createWriteStream(filepath);
+    await pipeline(response.data, writer);
+}
+
+function getHighestResSrc(srcset: string): string | null {
+    const entries = srcset.split(',').map(s => s.trim());
+    const last = entries[entries.length - 1];
+    if (!last) return null;
+    const url = last.split(' ')[0];
+    if (!url) return null;
+    return url.startsWith('//') ? 'https:' + url : url;
+}
+
+function toFilename(heading: string): string {
+    return heading
+        .replace(/\bIllustration\b/gi, '')
+        .trim()
+        .replace(/[\/\\]/g, '-')
+        .replace(/\s+/g, '-')
+        .replace(/-+/g, '-')
+        .toLowerCase();
+}
+
+async function getPageCount(): Promise<number> {
+    const { data } = await axios.get(TARGET_URL, {
+        headers: { 'User-Agent': 'Mozilla/5.0' }
+    });
+    const $ = cheerio.load(data);
+
+    const lastPage = $('.pagination__list li a')
+        .map((_, el) => parseInt($(el).text().trim()))
+        .get()
+        .filter(n => !isNaN(n))
+        .pop();
+
+    return lastPage ?? 1;
+}
+
+async function scrapePage(url: string): Promise<{ url: string; heading: string }[]> {
+    const { data } = await axios.get(url, {
+        headers: { 'User-Agent': 'Mozilla/5.0' }
+    });
+
+    const $ = cheerio.load(data);
+    const images: { url: string; heading: string }[] = [];
+
+    $('.card-wrapper').each((_, el) => {
+        const srcset = $(el).find('.card__media img').attr('srcset');
+        const heading = $(el).find('.card__heading.h5').text().trim();
+
+        if (srcset && heading) {
+            const url = getHighestResSrc(srcset);
+            if (url) images.push({ url, heading });
+        }
+    });
+
+    return images;
+}
+
+async function scrapeImages(): Promise<void> {
+    if (!fs.existsSync(OUTPUT_DIR)) fs.mkdirSync(OUTPUT_DIR);
+
+    const pageCount = await getPageCount();
+    console.log(`Found ${pageCount} pages`);
+
+    for (let page = 1; page <= pageCount; page++) {
+        const url = page === 1 ? TARGET_URL : `${TARGET_URL}?page=${page}`;
+        console.log(`\nScraping page ${page}/${pageCount}...`);
+
+        const images = await scrapePage(url);
+        console.log(`Found ${images.length} images`);
+
+        for (const { url: imgUrl, heading } of images) {
+            const ext = path.extname(new URL(imgUrl).pathname) || '.jpg';
+            const filename = toFilename(heading) + ext;
+
+            // Skip if already downloaded
+            if (fs.existsSync(path.join(OUTPUT_DIR, filename))) {
+                console.log(`⟳ Skipped (exists): ${filename}`);
+                continue;
+            }
+
+            try {
+                await downloadImage(imgUrl, filename);
+                console.log(`✓ ${filename}`);
+            } catch (err) {
+                console.error(`✗ Failed: ${imgUrl}`, (err as any).response?.status ?? err);
+            }
+        }
+    }
+
+    console.log('\nDone!');
+}
+
+scrapeImages();

package.json

@@ -0,0 +1,25 @@
+{
+  "name": "liveryscraper",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.ts",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "private": true,
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "type": "module",
+  "devDependencies": {
+    "@types/cheerio": "^0.22.35",
+    "@types/node": "^25.6.0",
+    "tsx": "^4.21.0",
+    "typescript": "^6.0.3"
+  },
+  "dependencies": {
+    "axios": "^1.15.1",
+    "cheerio": "^1.2.0",
+    "undici-types": "^7.19.2"
+  }
+}

tsconfig.json

@@ -0,0 +1,44 @@
+{
+  // Visit https://aka.ms/tsconfig to read more about this file
+  "compilerOptions": {
+    // File Layout
+    // "rootDir": "./src",
+    // "outDir": "./dist",
+
+    // Environment Settings
+    // See also https://aka.ms/tsconfig/module
+    "module": "nodenext",
+    "target": "esnext",
+    "types": [],
+    // For nodejs:
+    // "lib": ["esnext"],
+    // "types": ["node"],
+    // and npm install -D @types/node
+
+    // Other Outputs
+    "sourceMap": true,
+    "declaration": true,
+    "declarationMap": true,
+
+    // Stricter Typechecking Options
+    "noUncheckedIndexedAccess": true,
+    "exactOptionalPropertyTypes": true,
+
+    // Style Options
+    // "noImplicitReturns": true,
+    // "noImplicitOverride": true,
+    // "noUnusedLocals": true,
+    // "noUnusedParameters": true,
+    // "noFallthroughCasesInSwitch": true,
+    // "noPropertyAccessFromIndexSignature": true,
+
+    // Recommended Options
+    "strict": true,
+    "jsx": "react-jsx",
+    "verbatimModuleSyntax": true,
+    "isolatedModules": true,
+    "noUncheckedSideEffectImports": true,
+    "moduleDetection": "force",
+    "skipLibCheck": true,
+  }
+}