Add VM rebuild pipeline with isolated job chain and admin log API

2026-05-29 18:19:38 +07:00
parent 7ae237ee9d
commit 9c68273b4a
27 changed files with 2391 additions and 199 deletions
@@ -0,0 +1,103 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Jobs\VirtualMachine\Rebuild;
+
+use App\Http\Services\vCenter as VCenterService;
+use App\Models\ModuleVmware;
+use App\Models\vCenter as VCenterModel;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
+use Throwable;
+
+/**
+ * RebuildDeleteOldVmJob
+ *
+ * Final step of the rebuild pipeline. Runs only AFTER the DB has already
+ * been swapped to point at the new VM, so even if this job fails the
+ * customer's service is fully online on the new VM — the old VM just
+ * becomes an orphan that an operator can clean up later.
+ *
+ * Defensive guard: before deleting, re-read ModuleVmware.vmware_id from DB
+ * and refuse to delete if it still matches the "old" id we were told about.
+ * That would mean the swap somehow rolled back, and deleting would destroy
+ * the live VM.
+ */
+final class RebuildDeleteOldVmJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public int $tries   = 3;
+    public int $timeout = 180;
+    public int $backoff = 60;
+
+    public function __construct(
+        public readonly int $moduleVmwareId,
+        public readonly string $oldVmId,
+        public readonly ?int $reinstallLogId = null,
+    ) {
+    }
+
+    public function handle(VCenterService $vCenter): void
+    {
+        if ($this->oldVmId === '') {
+            Log::info('RebuildDeleteOldVmJob: nothing to delete (oldVmId empty)');
+            return;
+        }
+
+        $vm = ModuleVmware::find($this->moduleVmwareId);
+        if (!$vm) {
+            Log::warning('RebuildDeleteOldVmJob: ModuleVmware vanished, skipping', [
+                'module_vmware_id' => $this->moduleVmwareId,
+            ]);
+            return;
+        }
+
+        if ((string) $vm->vmware_id === $this->oldVmId) {
+            // DB swap never happened — refuse to delete the live VM.
+            Log::error('RebuildDeleteOldVmJob: ABORT — module_vmware still references oldVmId', [
+                'module_vmware_id' => $this->moduleVmwareId,
+                'old_vm_id'        => $this->oldVmId,
+            ]);
+            return;
+        }
+
+        $vcenter = VCenterModel::find($vm->vcenter_id);
+        if (!$vcenter || !$vCenter->loginServer($vcenter)) {
+            Log::error('RebuildDeleteOldVmJob: cannot login to vCenter', [
+                'old_vm_id' => $this->oldVmId,
+            ]);
+            return;
+        }
+
+        try {
+            $vCenter->PowerOffVirtualMachine($this->oldVmId);
+            $res = $vCenter->DeleteVirtualMachine($this->oldVmId);
+            Log::info('RebuildDeleteOldVmJob: old VM deleted', [
+                'old_vm_id' => $this->oldVmId,
+                'result'    => $res,
+            ]);
+        } catch (Throwable $e) {
+            Log::error('RebuildDeleteOldVmJob: delete failed', [
+                'old_vm_id' => $this->oldVmId,
+                'error'     => $e->getMessage(),
+            ]);
+            // Re-throw so Laravel retries.
+            throw $e;
+        }
+    }
+
+    public function failed(Throwable $e): void
+    {
+        Log::error('RebuildDeleteOldVmJob: retries exhausted — manual cleanup required', [
+            'module_vmware_id' => $this->moduleVmwareId,
+            'old_vm_id'        => $this->oldVmId,
+            'error'            => $e->getMessage(),
+        ]);
+    }
+}