chore: improved scraping algorithm by only adding one record per day

app/Console/Commands/DispatchUpdateManually.php

@@ -0,0 +1,43 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Jobs\UpdateCovidData;
+use Illuminate\Console\Command;
+
+class DispatchUpdateManually extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'job:dispatch';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Dispatches the update job manually';
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return int
+     */
+    public function handle()
+    {
+        dispatch(new UpdateCovidData);
+    }
+}

app/Jobs/UpdateCovidData.php

@@ -36,8 +36,11 @@ class UpdateCovidData implements ShouldQueue
         $url = 'https://api.apify.com/v2/datasets/Gm6qjTgGqxkEZTkuJ/items?format=json&clean=1';
         $res = Http::get($url);
         CovidData::whereNotNull('id')->delete();
+        $data_arr = [];
         foreach ($res->collect() as $data) {
+            //dd(Carbon::createFromFormat('Y-m-d', $data["lastUpdatedAtApify"]));
             try {
+                
                 $attr = [];
                 $attr["infected"] = $data["infected"] ?? null;
                 $attr["deceased"] = $data["deceased"] ?? null;
@@ -45,12 +48,25 @@ class UpdateCovidData implements ShouldQueue
                 $attr["quarantined"] = $data["quarantined"] ?? null;
                 $attr["tested"] = $data["tested"] ?? null;
                 $attr["created_at"] = Carbon::createFromFormat('Y-m-d\TH:i:s+', $data["lastUpdatedAtApify"]);
-                $d = new CovidData($attr);
-                $d->save();
+                $idx = Carbon::parse($data["lastUpdatedAtApify"])->format('Y-m-d');
+                if(!isset($data_arr[$idx])){
+                    $data_arr[$idx] = $attr;
+                }else if($attr["created_at"] > $data_arr[$idx]["created_at"]){
+                    $data_arr[$idx]["infected"] == $attr["infected"] ?? $data_arr[$idx]["infected"];
+                    $data_arr[$idx]["deceased"] == $attr["deceased"] ?? $data_arr[$idx]["deceased"];
+                    $data_arr[$idx]["recovered"] == $attr["recovered"] ?? $data_arr[$idx]["recovered"];
+                    $data_arr[$idx]["quarantined"] == $attr["quarantined"] ?? $data_arr[$idx]["quarantined"];
+                    $data_arr[$idx]["tested"] == $attr["tested"] ?? $data_arr[$idx]["tested"];
+                    $data_arr[$idx]["created_at"] = $attr["created_at"];
+                }
             } catch (\Exception $e) {
                 dd($data, $e);
             }
-        };
+        }
+        foreach($data_arr as $attr){
+            $d = new CovidData($attr);
+            $d->save();
+        }
         //dd(CovidData::all());
     }
 }