From b2f2d8b1048ac2623dc1c04c115f2b31c92ede14 Mon Sep 17 00:00:00 2001
From: josh <josh.dredge@outlook.com>
Date: Sat, 29 Aug 2026 00:17:41 +1000
Subject: [PATCH] Initial Commit

---
 app/Console/Commands/CachePostMedia.php       |  28 +++++
 app/Console/Commands/CacheStoryMedia.php      |  28 +++++
 app/Console/Commands/ScrapeFeed.php           |  56 ++++++---
 app/Console/Commands/ScrapeStories.php        |  47 +++++---
 app/Models/Post.php                           |   3 +
 app/Models/Story.php                          |   3 +
 app/Observers/PostObserver.php                |  18 +++
 app/Observers/StoryObserver.php               |  18 +++
 app/Services/MediaCacher.php                  | 108 ++++++++++++++++++
 app/Services/PostMediaCacher.php              |  49 ++++++++
 app/Services/StoryMediaCacher.php             |  27 +++++
 ...0_add_attachments_to_dm_messages_table.php |  24 ++++
 resources/js/lib/proxyImage.ts                |   4 +
 13 files changed, 378 insertions(+), 35 deletions(-)
 create mode 100644 app/Console/Commands/CachePostMedia.php
 create mode 100644 app/Console/Commands/CacheStoryMedia.php
 create mode 100644 app/Observers/PostObserver.php
 create mode 100644 app/Observers/StoryObserver.php
 create mode 100644 app/Services/MediaCacher.php
 create mode 100644 app/Services/PostMediaCacher.php
 create mode 100644 app/Services/StoryMediaCacher.php
 create mode 100644 database/migrations/2026_08_28_140000_add_attachments_to_dm_messages_table.php

diff --git a/app/Console/Commands/CachePostMedia.php b/app/Console/Commands/CachePostMedia.php
new file mode 100644
index 0000000..3c379c6
--- /dev/null
+++ b/app/Console/Commands/CachePostMedia.php
@@ -0,0 +1,28 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Post;
+use App\Services\PostMediaCacher;
+use Illuminate\Console\Attributes\Description;
+use Illuminate\Console\Attributes\Signature;
+use Illuminate\Console\Command;
+
+#[Signature('posts:cache-media')]
+#[Description('Download and locally cache any post media still pointing at Instagram\'s CDN')]
+class CachePostMedia extends Command
+{
+    public function handle(PostMediaCacher $mediaCacher): int
+    {
+        $count = 0;
+
+        foreach (Post::query()->cursor() as $post) {
+            $mediaCacher->cache($post);
+            $count++;
+        }
+
+        $this->info("Checked {$count} posts.");
+
+        return self::SUCCESS;
+    }
+}
diff --git a/app/Console/Commands/CacheStoryMedia.php b/app/Console/Commands/CacheStoryMedia.php
new file mode 100644
index 0000000..2f37ec0
--- /dev/null
+++ b/app/Console/Commands/CacheStoryMedia.php
@@ -0,0 +1,28 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Story;
+use App\Services\StoryMediaCacher;
+use Illuminate\Console\Attributes\Description;
+use Illuminate\Console\Attributes\Signature;
+use Illuminate\Console\Command;
+
+#[Signature('stories:cache-media')]
+#[Description('Download and locally cache any story media still pointing at Instagram\'s CDN')]
+class CacheStoryMedia extends Command
+{
+    public function handle(StoryMediaCacher $mediaCacher): int
+    {
+        $count = 0;
+
+        foreach (Story::query()->cursor() as $story) {
+            $mediaCacher->cache($story);
+            $count++;
+        }
+
+        $this->info("Checked {$count} stories.");
+
+        return self::SUCCESS;
+    }
+}
diff --git a/app/Console/Commands/ScrapeFeed.php b/app/Console/Commands/ScrapeFeed.php
index 9e7e422..60af38a 100644
--- a/app/Console/Commands/ScrapeFeed.php
+++ b/app/Console/Commands/ScrapeFeed.php
@@ -3,6 +3,7 @@
 namespace App\Console\Commands;
 
 use App\Models\Post;
+use App\Services\PostMediaCacher;
 use Illuminate\Console\Attributes\Description;
 use Illuminate\Console\Attributes\Signature;
 
@@ -21,28 +22,45 @@ class ScrapeFeed extends RunsPlaywrightScript
             return;
         }
 
+        $mediaCacher = app(PostMediaCacher::class);
         $posts = json_decode(file_get_contents($path), true);
 
         foreach ($posts as $post) {
-            Post::updateOrCreate(
-                ['id' => $post['id']],
-                [
-                    'code' => $post['code'],
-                    'username' => $post['username'],
-                    'full_name' => $post['full_name'],
-                    'caption' => $post['caption'],
-                    'accessibility_caption' => $post['accessibility_caption'],
-                    'is_video' => $post['is_video'],
-                    'is_carousel' => $post['is_carousel'],
-                    'like_count' => $post['like_count'],
-                    'comment_count' => $post['comment_count'],
-                    'taken_at' => $post['taken_at'],
-                    'image_url' => $post['image_url'],
-                    'video_url' => $post['video_url'],
-                    'carousel_images' => $post['carousel_images'],
-                    'profile_pic_url' => $post['profile_pic_url'],
-                ]
-            );
+            $attributes = [
+                'code' => $post['code'],
+                'username' => $post['username'],
+                'full_name' => $post['full_name'],
+                'caption' => $post['caption'],
+                'accessibility_caption' => $post['accessibility_caption'],
+                'is_video' => $post['is_video'],
+                'is_carousel' => $post['is_carousel'],
+                'like_count' => $post['like_count'],
+                'comment_count' => $post['comment_count'],
+                'taken_at' => $post['taken_at'],
+                'image_url' => $post['image_url'],
+                'video_url' => $post['video_url'],
+                'carousel_images' => $post['carousel_images'],
+                'profile_pic_url' => $post['profile_pic_url'],
+            ];
+
+            $existing = Post::find($post['id']);
+
+            // Once a post's media has been cached locally, its url columns
+            // no longer point at Instagram — don't let a re-scrape clobber
+            // them with a freshly (and just as temporarily) signed CDN url.
+            if ($existing) {
+                if (! $mediaCacher->isInstagramUrl($existing->image_url)) {
+                    unset($attributes['image_url']);
+                }
+                if (! $mediaCacher->isInstagramUrl($existing->video_url)) {
+                    unset($attributes['video_url']);
+                }
+                if ($existing->carousel_images && ! $mediaCacher->isInstagramUrl($existing->carousel_images[0]['url'] ?? null)) {
+                    unset($attributes['carousel_images']);
+                }
+            }
+
+            Post::updateOrCreate(['id' => $post['id']], $attributes);
         }
 
         $this->info('Upserted ' . count($posts) . ' posts.');
diff --git a/app/Console/Commands/ScrapeStories.php b/app/Console/Commands/ScrapeStories.php
index bff9c2a..7a678d0 100644
--- a/app/Console/Commands/ScrapeStories.php
+++ b/app/Console/Commands/ScrapeStories.php
@@ -3,6 +3,7 @@
 namespace App\Console\Commands;
 
 use App\Models\Story;
+use App\Services\StoryMediaCacher;
 use Illuminate\Console\Attributes\Description;
 use Illuminate\Console\Attributes\Signature;
 
@@ -21,25 +22,39 @@ class ScrapeStories extends RunsPlaywrightScript
             return;
         }
 
+        $mediaCacher = app(StoryMediaCacher::class);
         $items = json_decode(file_get_contents($path), true);
 
         foreach ($items as $item) {
-            Story::updateOrCreate(
-                ['id' => $item['id']],
-                [
-                    'user_id' => $item['user_id'],
-                    'username' => $item['username'],
-                    'profile_pic_url' => $item['profile_pic_url'],
-                    'code' => $item['code'],
-                    'is_video' => $item['is_video'],
-                    'taken_at' => $item['taken_at'],
-                    'expiring_at' => $item['expiring_at'],
-                    'image_url' => $item['image_url'],
-                    'video_url' => $item['video_url'],
-                    'accessibility_caption' => $item['accessibility_caption'],
-                    'muted' => $item['muted'],
-                ]
-            );
+            $attributes = [
+                'user_id' => $item['user_id'],
+                'username' => $item['username'],
+                'profile_pic_url' => $item['profile_pic_url'],
+                'code' => $item['code'],
+                'is_video' => $item['is_video'],
+                'taken_at' => $item['taken_at'],
+                'expiring_at' => $item['expiring_at'],
+                'image_url' => $item['image_url'],
+                'video_url' => $item['video_url'],
+                'accessibility_caption' => $item['accessibility_caption'],
+                'muted' => $item['muted'],
+            ];
+
+            $existing = Story::find($item['id']);
+
+            // Once a story's media has been cached locally, its url columns
+            // no longer point at Instagram — don't let a re-scrape clobber
+            // them with a freshly (and just as temporarily) signed CDN url.
+            if ($existing) {
+                if (! $mediaCacher->isInstagramUrl($existing->image_url)) {
+                    unset($attributes['image_url']);
+                }
+                if (! $mediaCacher->isInstagramUrl($existing->video_url)) {
+                    unset($attributes['video_url']);
+                }
+            }
+
+            Story::updateOrCreate(['id' => $item['id']], $attributes);
         }
 
         $this->info('Upserted ' . count($items) . ' story items.');
diff --git a/app/Models/Post.php b/app/Models/Post.php
index c24ddf4..0b9c410 100644
--- a/app/Models/Post.php
+++ b/app/Models/Post.php
@@ -2,10 +2,13 @@
 
 namespace App\Models;
 
+use App\Observers\PostObserver;
 use Carbon\Carbon;
+use Illuminate\Database\Eloquent\Attributes\ObservedBy;
 use Illuminate\Database\Eloquent\Casts\Attribute;
 use Illuminate\Database\Eloquent\Model;
 
+#[ObservedBy(PostObserver::class)]
 class Post extends Model
 {
     protected $table = 'posts';
diff --git a/app/Models/Story.php b/app/Models/Story.php
index b6f1eee..654a46d 100644
--- a/app/Models/Story.php
+++ b/app/Models/Story.php
@@ -2,8 +2,11 @@
 
 namespace App\Models;
 
+use App\Observers\StoryObserver;
+use Illuminate\Database\Eloquent\Attributes\ObservedBy;
 use Illuminate\Database\Eloquent\Model;
 
+#[ObservedBy(StoryObserver::class)]
 class Story extends Model
 {
     protected $table = 'stories';
diff --git a/app/Observers/PostObserver.php b/app/Observers/PostObserver.php
new file mode 100644
index 0000000..10c754a
--- /dev/null
+++ b/app/Observers/PostObserver.php
@@ -0,0 +1,18 @@
+<?php
+
+namespace App\Observers;
+
+use App\Models\Post;
+use App\Services\PostMediaCacher;
+
+class PostObserver
+{
+    public function __construct(private readonly PostMediaCacher $mediaCacher)
+    {
+    }
+
+    public function created(Post $post): void
+    {
+        $this->mediaCacher->cache($post);
+    }
+}
diff --git a/app/Observers/StoryObserver.php b/app/Observers/StoryObserver.php
new file mode 100644
index 0000000..d51eadd
--- /dev/null
+++ b/app/Observers/StoryObserver.php
@@ -0,0 +1,18 @@
+<?php
+
+namespace App\Observers;
+
+use App\Models\Story;
+use App\Services\StoryMediaCacher;
+
+class StoryObserver
+{
+    public function __construct(private readonly StoryMediaCacher $mediaCacher)
+    {
+    }
+
+    public function created(Story $story): void
+    {
+        $this->mediaCacher->cache($story);
+    }
+}
diff --git a/app/Services/MediaCacher.php b/app/Services/MediaCacher.php
new file mode 100644
index 0000000..6829797
--- /dev/null
+++ b/app/Services/MediaCacher.php
@@ -0,0 +1,108 @@
+<?php
+
+namespace App\Services;
+
+use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Storage;
+use Throwable;
+
+/**
+ * Shared logic for downloading a model's Instagram-hosted media into local
+ * storage and rewriting its url columns to point at the cached copy.
+ * Instagram's CDN urls are signed and expire, so caching is a one-way trip:
+ * once a field holds a local url it's left alone on every later call, which
+ * makes cache() safe to call repeatedly. It also repairs any url that was
+ * cached as an absolute address before switching to relative ones, without
+ * re-downloading — the file is already on disk.
+ */
+abstract class MediaCacher
+{
+    private const INSTAGRAM_HOST_SUFFIXES = ['fbcdn.net', 'cdninstagram.com'];
+
+    /**
+     * True if the url still points at Instagram's own CDN (i.e. hasn't been
+     * cached locally yet). Public so callers writing scraped data back to
+     * an existing row can check it before deciding whether to overwrite a
+     * field a cache pass already replaced.
+     */
+    public function isInstagramUrl(?string $url): bool
+    {
+        if (! $url) {
+            return false;
+        }
+
+        $host = parse_url($url, PHP_URL_HOST);
+
+        if (! $host) {
+            return false;
+        }
+
+        foreach (self::INSTAGRAM_HOST_SUFFIXES as $suffix) {
+            if (str_ends_with($host, $suffix)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Figures out what, if anything, a url field needs to become: null if
+     * it's already a relative path we generated (nothing to do), a
+     * rewritten relative path if it's an absolute url pointing at our own
+     * /storage/ (cached before the relative-path fix, no download needed),
+     * a freshly downloaded local path if it's still on Instagram's CDN, or
+     * null for anything else we don't recognize.
+     */
+    protected function resolveUrl(?string $url, string $pathWithoutExtension): ?string
+    {
+        if (! $url || str_starts_with($url, '/storage/')) {
+            return null;
+        }
+
+        if (($pos = strpos($url, '/storage/')) !== false) {
+            return substr($url, $pos);
+        }
+
+        if ($this->isInstagramUrl($url)) {
+            return $this->download($url, $pathWithoutExtension);
+        }
+
+        return null;
+    }
+
+    private function download(string $url, string $pathWithoutExtension): ?string
+    {
+        try {
+            $response = Http::withHeaders(['User-Agent' => 'Mozilla/5.0'])->timeout(30)->get($url);
+        } catch (Throwable $e) {
+            Log::warning(static::class.": failed to fetch {$url}: {$e->getMessage()}");
+
+            return null;
+        }
+
+        if (! $response->successful()) {
+            Log::warning(static::class.": got {$response->status()} fetching {$url}");
+
+            return null;
+        }
+
+        $path = $pathWithoutExtension.'.'.$this->guessExtension($url, $response->header('Content-Type'));
+
+        Storage::disk('public')->put($path, $response->body());
+
+        return '/storage/'.$path;
+    }
+
+    private function guessExtension(string $url, ?string $contentType): string
+    {
+        $fromUrl = pathinfo(parse_url($url, PHP_URL_PATH) ?? '', PATHINFO_EXTENSION);
+
+        if ($fromUrl && strlen($fromUrl) <= 4) {
+            return strtolower($fromUrl);
+        }
+
+        return str_contains((string) $contentType, 'video') ? 'mp4' : 'jpg';
+    }
+}
diff --git a/app/Services/PostMediaCacher.php b/app/Services/PostMediaCacher.php
new file mode 100644
index 0000000..c564f36
--- /dev/null
+++ b/app/Services/PostMediaCacher.php
@@ -0,0 +1,49 @@
+<?php
+
+namespace App\Services;
+
+use App\Models\Post;
+
+class PostMediaCacher extends MediaCacher
+{
+    public function cache(Post $post): void
+    {
+        $dirty = false;
+
+        if ($resolved = $this->resolveUrl($post->image_url, "posts/images/{$post->id}")) {
+            $post->image_url = $resolved;
+            $dirty = true;
+        }
+
+        if ($resolved = $this->resolveUrl($post->video_url, "posts/videos/{$post->id}")) {
+            $post->video_url = $resolved;
+            $dirty = true;
+        }
+
+        if ($carousel = $post->carousel_images) {
+            $changed = false;
+
+            foreach ($carousel as $i => &$item) {
+                if (! empty($item['url']) && $resolved = $this->resolveUrl($item['url'], "posts/carousel_images/{$post->id}/{$i}")) {
+                    $item['url'] = $resolved;
+                    $changed = true;
+                }
+
+                if (! empty($item['video_url']) && $resolved = $this->resolveUrl($item['video_url'], "posts/carousel_images/{$post->id}/{$i}-video")) {
+                    $item['video_url'] = $resolved;
+                    $changed = true;
+                }
+            }
+            unset($item);
+
+            if ($changed) {
+                $post->carousel_images = $carousel;
+                $dirty = true;
+            }
+        }
+
+        if ($dirty) {
+            $post->saveQuietly();
+        }
+    }
+}
diff --git a/app/Services/StoryMediaCacher.php b/app/Services/StoryMediaCacher.php
new file mode 100644
index 0000000..f66bd15
--- /dev/null
+++ b/app/Services/StoryMediaCacher.php
@@ -0,0 +1,27 @@
+<?php
+
+namespace App\Services;
+
+use App\Models\Story;
+
+class StoryMediaCacher extends MediaCacher
+{
+    public function cache(Story $story): void
+    {
+        $dirty = false;
+
+        if ($resolved = $this->resolveUrl($story->image_url, "stories/images/{$story->id}")) {
+            $story->image_url = $resolved;
+            $dirty = true;
+        }
+
+        if ($resolved = $this->resolveUrl($story->video_url, "stories/videos/{$story->id}")) {
+            $story->video_url = $resolved;
+            $dirty = true;
+        }
+
+        if ($dirty) {
+            $story->saveQuietly();
+        }
+    }
+}
diff --git a/database/migrations/2026_08_28_140000_add_attachments_to_dm_messages_table.php b/database/migrations/2026_08_28_140000_add_attachments_to_dm_messages_table.php
new file mode 100644
index 0000000..ce9b1a8
--- /dev/null
+++ b/database/migrations/2026_08_28_140000_add_attachments_to_dm_messages_table.php
@@ -0,0 +1,24 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    public function up(): void
+    {
+        Schema::table('dm_messages', function (Blueprint $table) {
+            // IMAGES messages can carry multiple photos in one message, so a
+            // single media_url column doesn't fit — store the full list.
+            $table->jsonb('attachments')->nullable();
+        });
+    }
+
+    public function down(): void
+    {
+        Schema::table('dm_messages', function (Blueprint $table) {
+            $table->dropColumn('attachments');
+        });
+    }
+};
diff --git a/resources/js/lib/proxyImage.ts b/resources/js/lib/proxyImage.ts
index a3577db..0208b0b 100644
--- a/resources/js/lib/proxyImage.ts
+++ b/resources/js/lib/proxyImage.ts
@@ -1,4 +1,8 @@
 export function proxyImage(url: string | null | undefined): string | undefined {
     if (!url) return undefined;
+    // Already a local path (e.g. cached media) — load it directly. The proxy
+    // only knows how to handle Instagram's own CDN hosts, and a relative
+    // path has no host at all, so it would otherwise get rejected.
+    if (url.startsWith('/')) return url;
     return `/image-proxy?url=${encodeURIComponent(url)}`;
 }