Initial Commit

app/Console/Commands/CachePostMedia.php

@@ -0,0 +1,28 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Post;
+use App\Services\PostMediaCacher;
+use Illuminate\Console\Attributes\Description;
+use Illuminate\Console\Attributes\Signature;
+use Illuminate\Console\Command;
+
+#[Signature('posts:cache-media')]
+#[Description('Download and locally cache any post media still pointing at Instagram\'s CDN')]
+class CachePostMedia extends Command
+{
+    public function handle(PostMediaCacher $mediaCacher): int
+    {
+        $count = 0;
+
+        foreach (Post::query()->cursor() as $post) {
+            $mediaCacher->cache($post);
+            $count++;
+        }
+
+        $this->info("Checked {$count} posts.");
+
+        return self::SUCCESS;
+    }
+}

app/Console/Commands/CacheStoryMedia.php

@@ -0,0 +1,28 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Story;
+use App\Services\StoryMediaCacher;
+use Illuminate\Console\Attributes\Description;
+use Illuminate\Console\Attributes\Signature;
+use Illuminate\Console\Command;
+
+#[Signature('stories:cache-media')]
+#[Description('Download and locally cache any story media still pointing at Instagram\'s CDN')]
+class CacheStoryMedia extends Command
+{
+    public function handle(StoryMediaCacher $mediaCacher): int
+    {
+        $count = 0;
+
+        foreach (Story::query()->cursor() as $story) {
+            $mediaCacher->cache($story);
+            $count++;
+        }
+
+        $this->info("Checked {$count} stories.");
+
+        return self::SUCCESS;
+    }
+}

app/Console/Commands/ScrapeFeed.php

@@ -3,6 +3,7 @@
 namespace App\Console\Commands;
 
 use App\Models\Post;
+use App\Services\PostMediaCacher;
 use Illuminate\Console\Attributes\Description;
 use Illuminate\Console\Attributes\Signature;
 
@@ -21,28 +22,45 @@ class ScrapeFeed extends RunsPlaywrightScript
             return;
         }
 
+        $mediaCacher = app(PostMediaCacher::class);
         $posts = json_decode(file_get_contents($path), true);
 
         foreach ($posts as $post) {
-            Post::updateOrCreate(
-                ['id' => $post['id']],
-                [
-                    'code' => $post['code'],
-                    'username' => $post['username'],
-                    'full_name' => $post['full_name'],
-                    'caption' => $post['caption'],
-                    'accessibility_caption' => $post['accessibility_caption'],
-                    'is_video' => $post['is_video'],
-                    'is_carousel' => $post['is_carousel'],
-                    'like_count' => $post['like_count'],
-                    'comment_count' => $post['comment_count'],
-                    'taken_at' => $post['taken_at'],
-                    'image_url' => $post['image_url'],
-                    'video_url' => $post['video_url'],
-                    'carousel_images' => $post['carousel_images'],
-                    'profile_pic_url' => $post['profile_pic_url'],
-                ]
-            );
+            $attributes = [
+                'code' => $post['code'],
+                'username' => $post['username'],
+                'full_name' => $post['full_name'],
+                'caption' => $post['caption'],
+                'accessibility_caption' => $post['accessibility_caption'],
+                'is_video' => $post['is_video'],
+                'is_carousel' => $post['is_carousel'],
+                'like_count' => $post['like_count'],
+                'comment_count' => $post['comment_count'],
+                'taken_at' => $post['taken_at'],
+                'image_url' => $post['image_url'],
+                'video_url' => $post['video_url'],
+                'carousel_images' => $post['carousel_images'],
+                'profile_pic_url' => $post['profile_pic_url'],
+            ];
+
+            $existing = Post::find($post['id']);
+
+            // Once a post's media has been cached locally, its url columns
+            // no longer point at Instagram — don't let a re-scrape clobber
+            // them with a freshly (and just as temporarily) signed CDN url.
+            if ($existing) {
+                if (! $mediaCacher->isInstagramUrl($existing->image_url)) {
+                    unset($attributes['image_url']);
+                }
+                if (! $mediaCacher->isInstagramUrl($existing->video_url)) {
+                    unset($attributes['video_url']);
+                }
+                if ($existing->carousel_images && ! $mediaCacher->isInstagramUrl($existing->carousel_images[0]['url'] ?? null)) {
+                    unset($attributes['carousel_images']);
+                }
+            }
+
+            Post::updateOrCreate(['id' => $post['id']], $attributes);
         }
 
         $this->info('Upserted ' . count($posts) . ' posts.');

app/Console/Commands/ScrapeStories.php

@@ -3,6 +3,7 @@
 namespace App\Console\Commands;
 
 use App\Models\Story;
+use App\Services\StoryMediaCacher;
 use Illuminate\Console\Attributes\Description;
 use Illuminate\Console\Attributes\Signature;
 
@@ -21,25 +22,39 @@ class ScrapeStories extends RunsPlaywrightScript
             return;
         }
 
+        $mediaCacher = app(StoryMediaCacher::class);
         $items = json_decode(file_get_contents($path), true);
 
         foreach ($items as $item) {
-            Story::updateOrCreate(
-                ['id' => $item['id']],
-                [
-                    'user_id' => $item['user_id'],
-                    'username' => $item['username'],
-                    'profile_pic_url' => $item['profile_pic_url'],
-                    'code' => $item['code'],
-                    'is_video' => $item['is_video'],
-                    'taken_at' => $item['taken_at'],
-                    'expiring_at' => $item['expiring_at'],
-                    'image_url' => $item['image_url'],
-                    'video_url' => $item['video_url'],
-                    'accessibility_caption' => $item['accessibility_caption'],
-                    'muted' => $item['muted'],
-                ]
-            );
+            $attributes = [
+                'user_id' => $item['user_id'],
+                'username' => $item['username'],
+                'profile_pic_url' => $item['profile_pic_url'],
+                'code' => $item['code'],
+                'is_video' => $item['is_video'],
+                'taken_at' => $item['taken_at'],
+                'expiring_at' => $item['expiring_at'],
+                'image_url' => $item['image_url'],
+                'video_url' => $item['video_url'],
+                'accessibility_caption' => $item['accessibility_caption'],
+                'muted' => $item['muted'],
+            ];
+
+            $existing = Story::find($item['id']);
+
+            // Once a story's media has been cached locally, its url columns
+            // no longer point at Instagram — don't let a re-scrape clobber
+            // them with a freshly (and just as temporarily) signed CDN url.
+            if ($existing) {
+                if (! $mediaCacher->isInstagramUrl($existing->image_url)) {
+                    unset($attributes['image_url']);
+                }
+                if (! $mediaCacher->isInstagramUrl($existing->video_url)) {
+                    unset($attributes['video_url']);
+                }
+            }
+
+            Story::updateOrCreate(['id' => $item['id']], $attributes);
         }
 
         $this->info('Upserted ' . count($items) . ' story items.');

app/Models/Post.php

@@ -2,10 +2,13 @@
 
 namespace App\Models;
 
+use App\Observers\PostObserver;
 use Carbon\Carbon;
+use Illuminate\Database\Eloquent\Attributes\ObservedBy;
 use Illuminate\Database\Eloquent\Casts\Attribute;
 use Illuminate\Database\Eloquent\Model;
 
+#[ObservedBy(PostObserver::class)]
 class Post extends Model
 {
     protected $table = 'posts';

app/Models/Story.php

@@ -2,8 +2,11 @@
 
 namespace App\Models;
 
+use App\Observers\StoryObserver;
+use Illuminate\Database\Eloquent\Attributes\ObservedBy;
 use Illuminate\Database\Eloquent\Model;
 
+#[ObservedBy(StoryObserver::class)]
 class Story extends Model
 {
     protected $table = 'stories';

app/Observers/PostObserver.php

@@ -0,0 +1,18 @@
+<?php
+
+namespace App\Observers;
+
+use App\Models\Post;
+use App\Services\PostMediaCacher;
+
+class PostObserver
+{
+    public function __construct(private readonly PostMediaCacher $mediaCacher)
+    {
+    }
+
+    public function created(Post $post): void
+    {
+        $this->mediaCacher->cache($post);
+    }
+}

app/Observers/StoryObserver.php

@@ -0,0 +1,18 @@
+<?php
+
+namespace App\Observers;
+
+use App\Models\Story;
+use App\Services\StoryMediaCacher;
+
+class StoryObserver
+{
+    public function __construct(private readonly StoryMediaCacher $mediaCacher)
+    {
+    }
+
+    public function created(Story $story): void
+    {
+        $this->mediaCacher->cache($story);
+    }
+}

app/Services/MediaCacher.php

@@ -0,0 +1,108 @@
+<?php
+
+namespace App\Services;
+
+use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Storage;
+use Throwable;
+
+/**
+ * Shared logic for downloading a model's Instagram-hosted media into local
+ * storage and rewriting its url columns to point at the cached copy.
+ * Instagram's CDN urls are signed and expire, so caching is a one-way trip:
+ * once a field holds a local url it's left alone on every later call, which
+ * makes cache() safe to call repeatedly. It also repairs any url that was
+ * cached as an absolute address before switching to relative ones, without
+ * re-downloading — the file is already on disk.
+ */
+abstract class MediaCacher
+{
+    private const INSTAGRAM_HOST_SUFFIXES = ['fbcdn.net', 'cdninstagram.com'];
+
+    /**
+     * True if the url still points at Instagram's own CDN (i.e. hasn't been
+     * cached locally yet). Public so callers writing scraped data back to
+     * an existing row can check it before deciding whether to overwrite a
+     * field a cache pass already replaced.
+     */
+    public function isInstagramUrl(?string $url): bool
+    {
+        if (! $url) {
+            return false;
+        }
+
+        $host = parse_url($url, PHP_URL_HOST);
+
+        if (! $host) {
+            return false;
+        }
+
+        foreach (self::INSTAGRAM_HOST_SUFFIXES as $suffix) {
+            if (str_ends_with($host, $suffix)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Figures out what, if anything, a url field needs to become: null if
+     * it's already a relative path we generated (nothing to do), a
+     * rewritten relative path if it's an absolute url pointing at our own
+     * /storage/ (cached before the relative-path fix, no download needed),
+     * a freshly downloaded local path if it's still on Instagram's CDN, or
+     * null for anything else we don't recognize.
+     */
+    protected function resolveUrl(?string $url, string $pathWithoutExtension): ?string
+    {
+        if (! $url || str_starts_with($url, '/storage/')) {
+            return null;
+        }
+
+        if (($pos = strpos($url, '/storage/')) !== false) {
+            return substr($url, $pos);
+        }
+
+        if ($this->isInstagramUrl($url)) {
+            return $this->download($url, $pathWithoutExtension);
+        }
+
+        return null;
+    }
+
+    private function download(string $url, string $pathWithoutExtension): ?string
+    {
+        try {
+            $response = Http::withHeaders(['User-Agent' => 'Mozilla/5.0'])->timeout(30)->get($url);
+        } catch (Throwable $e) {
+            Log::warning(static::class.": failed to fetch {$url}: {$e->getMessage()}");
+
+            return null;
+        }
+
+        if (! $response->successful()) {
+            Log::warning(static::class.": got {$response->status()} fetching {$url}");
+
+            return null;
+        }
+
+        $path = $pathWithoutExtension.'.'.$this->guessExtension($url, $response->header('Content-Type'));
+
+        Storage::disk('public')->put($path, $response->body());
+
+        return '/storage/'.$path;
+    }
+
+    private function guessExtension(string $url, ?string $contentType): string
+    {
+        $fromUrl = pathinfo(parse_url($url, PHP_URL_PATH) ?? '', PATHINFO_EXTENSION);
+
+        if ($fromUrl && strlen($fromUrl) <= 4) {
+            return strtolower($fromUrl);
+        }
+
+        return str_contains((string) $contentType, 'video') ? 'mp4' : 'jpg';
+    }
+}

app/Services/PostMediaCacher.php

@@ -0,0 +1,49 @@
+<?php
+
+namespace App\Services;
+
+use App\Models\Post;
+
+class PostMediaCacher extends MediaCacher
+{
+    public function cache(Post $post): void
+    {
+        $dirty = false;
+
+        if ($resolved = $this->resolveUrl($post->image_url, "posts/images/{$post->id}")) {
+            $post->image_url = $resolved;
+            $dirty = true;
+        }
+
+        if ($resolved = $this->resolveUrl($post->video_url, "posts/videos/{$post->id}")) {
+            $post->video_url = $resolved;
+            $dirty = true;
+        }
+
+        if ($carousel = $post->carousel_images) {
+            $changed = false;
+
+            foreach ($carousel as $i => &$item) {
+                if (! empty($item['url']) && $resolved = $this->resolveUrl($item['url'], "posts/carousel_images/{$post->id}/{$i}")) {
+                    $item['url'] = $resolved;
+                    $changed = true;
+                }
+
+                if (! empty($item['video_url']) && $resolved = $this->resolveUrl($item['video_url'], "posts/carousel_images/{$post->id}/{$i}-video")) {
+                    $item['video_url'] = $resolved;
+                    $changed = true;
+                }
+            }
+            unset($item);
+
+            if ($changed) {
+                $post->carousel_images = $carousel;
+                $dirty = true;
+            }
+        }
+
+        if ($dirty) {
+            $post->saveQuietly();
+        }
+    }
+}

app/Services/StoryMediaCacher.php

@@ -0,0 +1,27 @@
+<?php
+
+namespace App\Services;
+
+use App\Models\Story;
+
+class StoryMediaCacher extends MediaCacher
+{
+    public function cache(Story $story): void
+    {
+        $dirty = false;
+
+        if ($resolved = $this->resolveUrl($story->image_url, "stories/images/{$story->id}")) {
+            $story->image_url = $resolved;
+            $dirty = true;
+        }
+
+        if ($resolved = $this->resolveUrl($story->video_url, "stories/videos/{$story->id}")) {
+            $story->video_url = $resolved;
+            $dirty = true;
+        }
+
+        if ($dirty) {
+            $story->saveQuietly();
+        }
+    }
+}