diff --git a/app/Http/Controllers/admin/TaskController.php b/app/Http/Controllers/admin/TaskController.php index c524471..32cbc13 100644 --- a/app/Http/Controllers/admin/TaskController.php +++ b/app/Http/Controllers/admin/TaskController.php @@ -7,6 +7,7 @@ use App\Jobs\Hosting\CreateHostingJobs; use App\Jobs\VirtualMachine\Action\RemoveVirtualMachine; use App\Jobs\VirtualMachine\Action\VMJobs; use App\Jobs\VirtualMachine\Create\CreateVMJobs; +use App\Jobs\VirtualMachine\Handle\WaitStatusVirtualMachine; use App\Models\Log; use App\Repositories\Module\HostingServer\HostingServerRepositoryInterface; use App\Repositories\Module\IpAddress\IpAddressRepositoryInterface; @@ -170,9 +171,20 @@ class TaskController extends Controller return back()->withErrors($validator); } - $jobs = new VMJobs($request->product, $request->action); + // ponytail: map action → waitFor (operations_ready value to poll WHILE waiting) + $waitMap = [ + 'Suspend' => true, + 'ShutdownGuest' => true, + 'PowerOff' => true, + 'Reboot' => false, + 'Start' => false, + ]; - dispatch($jobs)->delay(now()->addSeconds(5)); + dispatch(new VMJobs($request->product, $request->action)); + + if (isset($waitMap[$request->action])) { + dispatch(new WaitStatusVirtualMachine($request->product, $waitMap[$request->action])); + } Log::create([ 'agent' => $request->userAgent(), diff --git a/app/Jobs/VirtualMachine/Action/VMJobs.php b/app/Jobs/VirtualMachine/Action/VMJobs.php index 412546a..99040d6 100644 --- a/app/Jobs/VirtualMachine/Action/VMJobs.php +++ b/app/Jobs/VirtualMachine/Action/VMJobs.php @@ -6,7 +6,6 @@ use App\Events\NotificationStatus; use App\Http\Services\PusherNotification; use App\Http\Services\Telegram; use App\Http\Services\vCenter as ServiceVCenter; -use App\Jobs\VirtualMachine\Handle\WaitStatusVirtualMachine; use App\Models\ModuleVmware; use App\Models\vCenter; use Illuminate\Bus\Queueable; @@ -29,9 +28,8 @@ class VMJobs implements ShouldQueue protected $update = true; - public function __construct($product = 0, $action = '',$update = true) + public function __construct($product = 0, $action = '', $update = true) { - $this->data = $product; $this->push = new PusherNotification; $this->telegram = new Telegram; @@ -42,137 +40,111 @@ class VMJobs implements ShouldQueue public function handle(ServiceVCenter $serviceVCenter) { - $product = ModuleVmware::with([ 'Customer', 'IpAddress.Host.Datacenter.vCenter' ])->find($this->data); - $serviceVCenter->token = $product->IpAddress->Host->Datacenter->vCenter->token; - $serviceVCenter->apiUri = $product->IpAddress->Host->Datacenter->vCenter->host; + if (!$product || !$product->IpAddress?->Host?->Datacenter?->vCenter) { + throw new \Exception("VMJobs: product {$this->data} missing vCenter relationship"); + } + + $vcenter = $product->IpAddress->Host->Datacenter->vCenter; + $serviceVCenter->setConnection($vcenter->host, $vcenter->token); $VirtualMachineStatus = null; $status = 1; - $jobs = null; - switch ($this->action) - { + + switch ($this->action) { case "Suspend": $VirtualMachineStatus = $serviceVCenter->SuspendVirtualMachine($product->vmware_id); - if ($VirtualMachineStatus === "Login_Again") - { + if ($VirtualMachineStatus === "Login_Again") { $token = $serviceVCenter->loginServer($product->IpAddress->Host->Datacenter->vCenter); - if ($token) - { - vCenter::where('id',$product->IpAddress->Host->Datacenter->vCenter->id)->update(['token' => $token]); + if ($token) { + vCenter::where('id', $product->IpAddress->Host->Datacenter->vCenter->id)->update(['token' => $token]); $serviceVCenter->token = $token; } $VirtualMachineStatus = $serviceVCenter->SuspendVirtualMachine($product->vmware_id); } $status = 2; - $jobs = new WaitStatusVirtualMachine($product->vmware_id,$serviceVCenter,$product->customer_id,true); break; + case "Reboot": $VirtualMachineStatus = $serviceVCenter->RebootVirtualMachine($product->vmware_id); - - if ($VirtualMachineStatus === "Login_Again") - { + if ($VirtualMachineStatus === "Login_Again") { $token = $serviceVCenter->loginServer($product->IpAddress->Host->Datacenter->vCenter); - if ($token) - { - vCenter::where('id',$product->IpAddress->Host->Datacenter->vCenter->id)->update(['token' => $token]); + if ($token) { + vCenter::where('id', $product->IpAddress->Host->Datacenter->vCenter->id)->update(['token' => $token]); $serviceVCenter->token = $token; } $VirtualMachineStatus = $serviceVCenter->RebootVirtualMachine($product->vmware_id); } - $jobs = new WaitStatusVirtualMachine($product->vmware_id,$serviceVCenter,$product->customer_id,false); break; + case "Start": $VirtualMachineStatus = $serviceVCenter->CheckVirtualMachine($product->vmware_id); - - if ($VirtualMachineStatus === "Login_Again") - { + if ($VirtualMachineStatus === "Login_Again") { $token = $serviceVCenter->loginServer($product->IpAddress->Host->Datacenter->vCenter); - if ($token) - { - vCenter::where('id',$product->IpAddress->Host->Datacenter->vCenter->id)->update(['token' => $token]); + if ($token) { + vCenter::where('id', $product->IpAddress->Host->Datacenter->vCenter->id)->update(['token' => $token]); $serviceVCenter->token = $token; } $VirtualMachineStatus = $serviceVCenter->CheckVirtualMachine($product->vmware_id); } - if ($VirtualMachineStatus['data']['operations_ready'] === true) - { + if ($VirtualMachineStatus['data']['operations_ready'] === true) { break; } $VirtualMachineStatus = $serviceVCenter->StartVirtualMachine($product->vmware_id); - $jobs = new WaitStatusVirtualMachine($product->vmware_id,$serviceVCenter,$product->customer_id,false); break; + case "ShutdownGuest": $VirtualMachineStatus = $serviceVCenter->CheckVirtualMachine($product->vmware_id); - - if ($VirtualMachineStatus === "Login_Again") - { + if ($VirtualMachineStatus === "Login_Again") { $token = $serviceVCenter->loginServer($product->IpAddress->Host->Datacenter->vCenter); - if ($token) - { - vCenter::where('id',$product->IpAddress->Host->Datacenter->vCenter->id) - ->update(['token' => $token]); + if ($token) { + vCenter::where('id', $product->IpAddress->Host->Datacenter->vCenter->id)->update(['token' => $token]); $serviceVCenter->token = $token; } $VirtualMachineStatus = $serviceVCenter->CheckVirtualMachine($product->vmware_id); } - if ($VirtualMachineStatus['data']['operations_ready'] === false) - { + if ($VirtualMachineStatus['data']['operations_ready'] === false) { $status = 0; break; } $VirtualMachineStatus = $serviceVCenter->ShutdownGuestVirtualMachine($product->vmware_id); - - $jobs = new WaitStatusVirtualMachine($product->vmware_id,$serviceVCenter,$product->customer_id,true); - $status = 0; break; + case "PowerOff": $VirtualMachineStatus = $serviceVCenter->CheckVirtualMachine($product->vmware_id); - - if ($VirtualMachineStatus === "Login_Again") - { + if ($VirtualMachineStatus === "Login_Again") { $token = $serviceVCenter->loginServer($product->IpAddress->Host->Datacenter->vCenter); - if ($token) - { - vCenter::where('id',$product->IpAddress->Host->Datacenter->vCenter->id)->update(['token' => $token]); + if ($token) { + vCenter::where('id', $product->IpAddress->Host->Datacenter->vCenter->id)->update(['token' => $token]); $serviceVCenter->token = $token; } $VirtualMachineStatus = $serviceVCenter->CheckVirtualMachine($product->vmware_id); } - if ($VirtualMachineStatus['data']['operations_ready'] === false) - { + if ($VirtualMachineStatus['data']['operations_ready'] === false) { $status = 0; break; } $VirtualMachineStatus = $serviceVCenter->PowerOffVirtualMachine($product->vmware_id); - $jobs = new WaitStatusVirtualMachine($product->vmware_id,$serviceVCenter,$product->customer_id,true); $status = 0; break; } - if ($VirtualMachineStatus && $VirtualMachineStatus['error']) - { + if ($VirtualMachineStatus && $VirtualMachineStatus['error']) { throw new \Exception(json_encode($VirtualMachineStatus['data'])); } - if ($this->update) - { - $product->update( - ['status' => $status] - ); + if ($this->update) { + $product->update(['status' => $status]); } - dispatch($jobs)->delay(now()->addSeconds(5)); - } + public function failed(Throwable $exception) { - Event(new NotificationStatus("Notification: " . $exception->getMessage(),false)); - + Event(new NotificationStatus("Notification: " . $exception->getMessage(), false)); $this->telegram->SendErrorMessage($exception->getMessage()); } - } diff --git a/app/Jobs/VirtualMachine/Create/CreateVirtualMachineJob.php b/app/Jobs/VirtualMachine/Create/CreateVirtualMachineJob.php index 72b04cb..8a503e7 100644 --- a/app/Jobs/VirtualMachine/Create/CreateVirtualMachineJob.php +++ b/app/Jobs/VirtualMachine/Create/CreateVirtualMachineJob.php @@ -7,6 +7,7 @@ namespace App\Jobs\VirtualMachine\Create; use App\Http\Services\vCenter; use App\Jobs\VirtualMachine\Provision\AssignIpToGuestJob; use App\Models\OrderItem; +use App\Models\VmProvisionLock; use App\Services\VirtualMachine\VmAddonResolver; use App\Services\VirtualMachine\VmInfrastructureResolver; use App\Services\VirtualMachine\VmIpReservationService; @@ -55,8 +56,15 @@ class CreateVirtualMachineJob implements ShouldQueue ): bool { $reservation = null; $ipPoolId = null; + $lock = null; try { + $customerId = $this->item->order->customer_id; + $lock = VmProvisionLock::updateOrCreate( + ['order_item_id' => $this->item->id], + ['customer_id' => $customerId, 'status' => 'running'], + ); + $vmLogger->step('Create VM job started', [ 'order_item_id' => $this->item->id, 'is_retry' => $this->isRetry, @@ -198,12 +206,16 @@ class CreateVirtualMachineJob implements ShouldQueue 'vm_id' => $vmId, ]); + $lock?->update(['status' => 'success']); + return true; } catch (Throwable $e) { if ($reservation) { $ipReservation->release($reservation, $ipPoolId); } + $lock?->update(['status' => 'failed']); + $vmLogger->exception('Create VM job failed', $e, [ 'order_item_id' => $this->item->id, 'is_retry' => $this->isRetry, diff --git a/app/Jobs/VirtualMachine/Handle/WaitStatusVirtualMachine.php b/app/Jobs/VirtualMachine/Handle/WaitStatusVirtualMachine.php index 89424ad..c80cd30 100644 --- a/app/Jobs/VirtualMachine/Handle/WaitStatusVirtualMachine.php +++ b/app/Jobs/VirtualMachine/Handle/WaitStatusVirtualMachine.php @@ -3,6 +3,8 @@ namespace App\Jobs\VirtualMachine\Handle; use App\Events\VirtualMachine_Panel_Status; +use App\Http\Services\vCenter as ServiceVCenter; +use App\Models\ModuleVmware; use Illuminate\Bus\Queueable; use Illuminate\Console\Scheduling\Event; use Illuminate\Contracts\Queue\ShouldQueue; @@ -15,54 +17,49 @@ class WaitStatusVirtualMachine implements ShouldQueue { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - protected string $vmware_id; - protected $serviceVCetner; + protected int $productId; protected bool $status; - protected int $user; - /** - * Create a new job instance. - * - * @return void - */ - public function __construct($vmware_id,$service,$user,$status) + + public function __construct(int $productId, bool $status) { - // - $this->vmware_id = $vmware_id; - $this->serviceVCetner = $service; - $this->user = $user; + $this->productId = $productId; $this->status = $status; } - /** - * Execute the job. - * - * @return void - */ - public function handle() + public function handle(ServiceVCenter $serviceVCenter) { - Log::info("WaitStatusVirtualMachine: Started for VM {$this->vmware_id}, Target Status: " . ($this->status ? 'True' : 'False')); + $product = ModuleVmware::with(['IpAddress.Host.Datacenter.vCenter'])->find($this->productId); - $status = $this->serviceVCetner->CheckVirtualMachine($this->vmware_id); + if (!$product || !$product->IpAddress?->Host?->Datacenter?->vCenter) { + Log::error("WaitStatusVirtualMachine: Product {$this->productId} missing vCenter relationship"); + return; + } + + $vcenter = $product->IpAddress->Host->Datacenter->vCenter; + $serviceVCenter->setConnection($vcenter->host, $vcenter->token); + + $vmwareId = $product->vmware_id; + $userId = $product->customer_id; + + Log::info("WaitStatusVirtualMachine: Started for VM {$vmwareId}, Target Status: " . ($this->status ? 'True' : 'False')); + + $status = $serviceVCenter->CheckVirtualMachine($vmwareId); if ($status['error'] === true) { - Log::error("WaitStatusVirtualMachine: Error checking status for VM {$this->vmware_id}: " . json_encode($status['message'])); + Log::error("WaitStatusVirtualMachine: Error checking status for VM {$vmwareId}: " . json_encode($status['message'])); throw new \Exception(json_encode($status['message'])); } Log::info("WaitStatusVirtualMachine: Initial operations_ready: " . ($status['data']['operations_ready'] ? 'True' : 'False')); - // Wait loop: "Wait while operations_ready matches $this->status" - // Example: Stop Job passes true. We wait while it is true. When it becomes false (Off), we exit. if ($status['data']['operations_ready'] === $this->status) { Log::info("WaitStatusVirtualMachine: Entering wait loop..."); do { sleep(10); - $status = $this->serviceVCetner->CheckVirtualMachine($this->vmware_id); + $status = $serviceVCenter->CheckVirtualMachine($vmwareId); if ($status['error'] === true) { Log::warning("WaitStatusVirtualMachine: Failed to check status in loop. Breaking."); - // If check fails, maybe network glitch? We can break or continue. - // Let's break to avoid infinite loop on error break; } @@ -73,17 +70,11 @@ class WaitStatusVirtualMachine implements ShouldQueue Log::info("WaitStatusVirtualMachine: Skipping wait loop, condition already met."); } - // Action Completed - // 1. Clear Cache again (to ensure next fetch gets new state) - \Illuminate\Support\Facades\Cache::forget(\App\Support\Cache\CacheKey::build('vm', 'info', ['id' => $this->vmware_id])); + \Illuminate\Support\Facades\Cache::forget(\App\Support\Cache\CacheKey::build('vm', 'info', ['id' => $vmwareId])); - // 2. Update DB Status and Dispatch Event - $product = \App\Models\ModuleVmware::where('vmware_id', $this->vmware_id)->first(); + $product->refresh(); if ($product) { - Log::info("WaitStatusVirtualMachine: Product found. Updating status..."); - // If waiting for True (On) to become False (Off) -> New Status 0 (Off) - // If waiting for False (Off) to become True (On) -> New Status 1 (Active) $newStatus = ($this->status === true) ? 0 : 1; Log::info("WaitStatusVirtualMachine: Setting status to $newStatus"); @@ -96,19 +87,9 @@ class WaitStatusVirtualMachine implements ShouldQueue $statusStr = ($newStatus == 1) ? 'running' : 'stopped'; - Log::info("WaitStatusVirtualMachine: Dispatching VirtualMachineStatusChanged event."); - - \App\Events\VirtualMachineStatusChanged::dispatch( - $this->user, - $this->vmware_id, - $statusStr, - $msg - ); - } else { - Log::error("WaitStatusVirtualMachine: Product not found for vmware_id: {$this->vmware_id}"); + \App\Events\VirtualMachineStatusChanged::dispatch($userId, $vmwareId, $statusStr, $msg); } - // Legacy event (optional, keep to avoid breaking other things if they listen to it) - Event(new VirtualMachine_Panel_Status('Yêu cầu được xử lý thành công', $this->user)); + Event(new VirtualMachine_Panel_Status('Yêu cầu được xử lý thành công', $userId)); } } diff --git a/app/Jobs/VirtualMachine/Power/RebootVmJob.php b/app/Jobs/VirtualMachine/Power/RebootVmJob.php index 0b43a29..e88f961 100644 --- a/app/Jobs/VirtualMachine/Power/RebootVmJob.php +++ b/app/Jobs/VirtualMachine/Power/RebootVmJob.php @@ -71,12 +71,7 @@ class RebootVmJob implements ShouldQueue // In VMJobs: $jobs = new WaitStatusVirtualMachine($product->vmware_id,$serviceVCenter,$product->customer_id,false); - dispatch(new WaitStatusVirtualMachine( - $product->vmware_id, - $vCenterService, - $this->userId, - false - ))->delay(now()->addSeconds(5)); + dispatch(new WaitStatusVirtualMachine($this->productId, false))->delay(now()->addSeconds(5)); } else { $msg = is_array($result) ? ($result['data'] ?? 'Unknown error') : $result; diff --git a/app/Jobs/VirtualMachine/Power/StartVmJob.php b/app/Jobs/VirtualMachine/Power/StartVmJob.php index 0f90aa7..7161e1e 100644 --- a/app/Jobs/VirtualMachine/Power/StartVmJob.php +++ b/app/Jobs/VirtualMachine/Power/StartVmJob.php @@ -81,12 +81,7 @@ class StartVmJob implements ShouldQueue // For Start: // $jobs = new WaitStatusVirtualMachine($product->vmware_id, $serviceVCenter, $product->customer_id, false); - dispatch(new WaitStatusVirtualMachine( - $product->vmware_id, - $vCenterService, - $this->userId, - false // Expect operations_ready = false initially? Logic in WaitStatus is weird. - ))->delay(now()->addSeconds(5)); + dispatch(new WaitStatusVirtualMachine($this->productId, false))->delay(now()->addSeconds(5)); } else { $msg = is_array($result) ? ($result['data'] ?? 'Unknown error') : $result; diff --git a/app/Jobs/VirtualMachine/Power/StopVmJob.php b/app/Jobs/VirtualMachine/Power/StopVmJob.php index 37c670c..710e7f0 100644 --- a/app/Jobs/VirtualMachine/Power/StopVmJob.php +++ b/app/Jobs/VirtualMachine/Power/StopVmJob.php @@ -69,12 +69,7 @@ class StopVmJob implements ShouldQueue $product->update(['status' => 3]); // Processing - dispatch(new WaitStatusVirtualMachine( - $product->vmware_id, - $vCenterService, - $this->userId, - true - ))->delay(now()->addSeconds(5)); + dispatch(new WaitStatusVirtualMachine($this->productId, true))->delay(now()->addSeconds(5)); } else { $msg = is_array($result) ? ($result['data'] ?? 'Unknown error') : $result; diff --git a/app/Models/VmProvisionLock.php b/app/Models/VmProvisionLock.php new file mode 100644 index 0000000..794bd15 --- /dev/null +++ b/app/Models/VmProvisionLock.php @@ -0,0 +1,15 @@ +where('status', 'running')->exists(); + } +} diff --git a/app/Services/Api/Order/OrderService.php b/app/Services/Api/Order/OrderService.php index 89c23e4..b480566 100644 --- a/app/Services/Api/Order/OrderService.php +++ b/app/Services/Api/Order/OrderService.php @@ -6,6 +6,7 @@ use App\Enums\Order\OrderStatus; use App\Helpers\OrderHelper; use App\Jobs\Mail\Admin\AdminOrderNotificationJob; use App\Jobs\VirtualMachine\Create\CreateVirtualMachineJob; +use App\Models\VmProvisionLock; use App\Models\Addon; use App\Models\Log as CustomerActivityLog; use App\Models\Order; @@ -224,6 +225,14 @@ class OrderService private function dispatchVmJob(Order $order, OrderItem $item, Service $service, $customer): void { try { + if (VmProvisionLock::isRunning($customer->id)) { + Log::warning('dispatchVmJob: blocked — customer already has a running VM provision', [ + 'customer_id' => $customer->id, + 'order_item_id' => $item->id, + ]); + return; + } + dispatch(new CreateVirtualMachineJob($item)); Log::info('dispatchVmJob: CreateVirtualMachineJob dispatched', [ diff --git a/app/Services/VirtualMachine/VmIpReservationService.php b/app/Services/VirtualMachine/VmIpReservationService.php index bf50fd3..c44c6c4 100644 --- a/app/Services/VirtualMachine/VmIpReservationService.php +++ b/app/Services/VirtualMachine/VmIpReservationService.php @@ -24,7 +24,9 @@ class VmIpReservationService public function reserve(int $networkId, int $hostId, string $dns): ?array { return DB::transaction(function () use ($networkId, $hostId, $dns) { + // ponytail: exclude only IPs with an actual VM assigned, not stale __pending__ reservations $usedIps = ModuleIpAddress::where('module_network_id', $networkId) + ->where('vm_name', '!=', '__pending__') ->lockForUpdate() ->pluck('ip') ->toArray(); diff --git a/database/migrations/2026_07_13_000001_create_vm_provision_locks_table.php b/database/migrations/2026_07_13_000001_create_vm_provision_locks_table.php new file mode 100644 index 0000000..9384834 --- /dev/null +++ b/database/migrations/2026_07_13_000001_create_vm_provision_locks_table.php @@ -0,0 +1,24 @@ +id(); + $table->unsignedBigInteger('customer_id')->index(); + $table->unsignedBigInteger('order_item_id')->unique(); + $table->enum('status', ['running', 'success', 'failed'])->default('running'); + $table->timestamps(); + }); + } + + public function down(): void + { + Schema::dropIfExists('vm_provision_locks'); + } +};