bao/portal-api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement VM provision locking mechanism and enhance VM job handling

Browse Source
This commit is contained in:
baodev0
2026-07-13 11:23:53 +07:00
parent 0976e23d1a
commit 8d3410f8cc
11 changed files with 143 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Http/Controllers/admin/TaskController.php
+14 -2
View File
@@ -7,6 +7,7 @@ use App\Jobs\Hosting\CreateHostingJobs;
use App\Jobs\VirtualMachine\Action\RemoveVirtualMachine;
use App\Jobs\VirtualMachine\Action\VMJobs;
use App\Jobs\VirtualMachine\Create\CreateVMJobs;
use App\Jobs\VirtualMachine\Handle\WaitStatusVirtualMachine;
use App\Models\Log;
use App\Repositories\Module\HostingServer\HostingServerRepositoryInterface;
use App\Repositories\Module\IpAddress\IpAddressRepositoryInterface;
@@ -170,9 +171,20 @@ class TaskController extends Controller
return back()->withErrors($validator);
}
$jobs = new VMJobs($request->product, $request->action);
// ponytail: map action → waitFor (operations_ready value to poll WHILE waiting)
$waitMap = [
'Suspend' => true,
'ShutdownGuest' => true,
'PowerOff' => true,
'Reboot' => false,
'Start' => false,
];
dispatch($jobs)->delay(now()->addSeconds(5));
dispatch(new VMJobs($request->product, $request->action));
if (isset($waitMap[$request->action])) {
dispatch(new WaitStatusVirtualMachine($request->product, $waitMap[$request->action]));
}
Log::create([
'agent' => $request->userAgent(),
app/Jobs/VirtualMachine/Action/VMJobs.php
+36 -64
View File
@@ -6,7 +6,6 @@ use App\Events\NotificationStatus;
use App\Http\Services\PusherNotification;
use App\Http\Services\Telegram;
use App\Http\Services\vCenter as ServiceVCenter;
use App\Jobs\VirtualMachine\Handle\WaitStatusVirtualMachine;
use App\Models\ModuleVmware;
use App\Models\vCenter;
use Illuminate\Bus\Queueable;
@@ -29,9 +28,8 @@ class VMJobs implements ShouldQueue
protected $update = true;
public function __construct($product = 0, $action = '',$update = true)
public function __construct($product = 0, $action = '', $update = true)
{
$this->data = $product;
$this->push = new PusherNotification;
$this->telegram = new Telegram;
@@ -42,137 +40,111 @@ class VMJobs implements ShouldQueue
public function handle(ServiceVCenter $serviceVCenter)
{
$product = ModuleVmware::with([
'Customer',
'IpAddress.Host.Datacenter.vCenter'
])->find($this->data);
$serviceVCenter->token = $product->IpAddress->Host->Datacenter->vCenter->token;
$serviceVCenter->apiUri = $product->IpAddress->Host->Datacenter->vCenter->host;
if (!$product || !$product->IpAddress?->Host?->Datacenter?->vCenter) {
throw new \Exception("VMJobs: product {$this->data} missing vCenter relationship");
}
$vcenter = $product->IpAddress->Host->Datacenter->vCenter;
$serviceVCenter->setConnection($vcenter->host, $vcenter->token);
$VirtualMachineStatus = null;
$status = 1;
$jobs = null;
switch ($this->action)
{
switch ($this->action) {
case "Suspend":
$VirtualMachineStatus = $serviceVCenter->SuspendVirtualMachine($product->vmware_id);
if ($VirtualMachineStatus === "Login_Again")
{
if ($VirtualMachineStatus === "Login_Again") {
$token = $serviceVCenter->loginServer($product->IpAddress->Host->Datacenter->vCenter);
if ($token)
{
vCenter::where('id',$product->IpAddress->Host->Datacenter->vCenter->id)->update(['token' => $token]);
if ($token) {
vCenter::where('id', $product->IpAddress->Host->Datacenter->vCenter->id)->update(['token' => $token]);
$serviceVCenter->token = $token;
}
$VirtualMachineStatus = $serviceVCenter->SuspendVirtualMachine($product->vmware_id);
}
$status = 2;
$jobs = new WaitStatusVirtualMachine($product->vmware_id,$serviceVCenter,$product->customer_id,true);
break;
case "Reboot":
$VirtualMachineStatus = $serviceVCenter->RebootVirtualMachine($product->vmware_id);
if ($VirtualMachineStatus === "Login_Again")
{
if ($VirtualMachineStatus === "Login_Again") {
$token = $serviceVCenter->loginServer($product->IpAddress->Host->Datacenter->vCenter);
if ($token)
{
vCenter::where('id',$product->IpAddress->Host->Datacenter->vCenter->id)->update(['token' => $token]);
if ($token) {
vCenter::where('id', $product->IpAddress->Host->Datacenter->vCenter->id)->update(['token' => $token]);
$serviceVCenter->token = $token;
}
$VirtualMachineStatus = $serviceVCenter->RebootVirtualMachine($product->vmware_id);
}
$jobs = new WaitStatusVirtualMachine($product->vmware_id,$serviceVCenter,$product->customer_id,false);
break;
case "Start":
$VirtualMachineStatus = $serviceVCenter->CheckVirtualMachine($product->vmware_id);
if ($VirtualMachineStatus === "Login_Again")
{
if ($VirtualMachineStatus === "Login_Again") {
$token = $serviceVCenter->loginServer($product->IpAddress->Host->Datacenter->vCenter);
if ($token)
{
vCenter::where('id',$product->IpAddress->Host->Datacenter->vCenter->id)->update(['token' => $token]);
if ($token) {
vCenter::where('id', $product->IpAddress->Host->Datacenter->vCenter->id)->update(['token' => $token]);
$serviceVCenter->token = $token;
}
$VirtualMachineStatus = $serviceVCenter->CheckVirtualMachine($product->vmware_id);
}
if ($VirtualMachineStatus['data']['operations_ready'] === true)
{
if ($VirtualMachineStatus['data']['operations_ready'] === true) {
break;
}
$VirtualMachineStatus = $serviceVCenter->StartVirtualMachine($product->vmware_id);
$jobs = new WaitStatusVirtualMachine($product->vmware_id,$serviceVCenter,$product->customer_id,false);
break;
case "ShutdownGuest":
$VirtualMachineStatus = $serviceVCenter->CheckVirtualMachine($product->vmware_id);
if ($VirtualMachineStatus === "Login_Again")
{
if ($VirtualMachineStatus === "Login_Again") {
$token = $serviceVCenter->loginServer($product->IpAddress->Host->Datacenter->vCenter);
if ($token)
{
vCenter::where('id',$product->IpAddress->Host->Datacenter->vCenter->id)
->update(['token' => $token]);
if ($token) {
vCenter::where('id', $product->IpAddress->Host->Datacenter->vCenter->id)->update(['token' => $token]);
$serviceVCenter->token = $token;
}
$VirtualMachineStatus = $serviceVCenter->CheckVirtualMachine($product->vmware_id);
}
if ($VirtualMachineStatus['data']['operations_ready'] === false)
{
if ($VirtualMachineStatus['data']['operations_ready'] === false) {
$status = 0;
break;
}
$VirtualMachineStatus = $serviceVCenter->ShutdownGuestVirtualMachine($product->vmware_id);
$jobs = new WaitStatusVirtualMachine($product->vmware_id,$serviceVCenter,$product->customer_id,true);
$status = 0;
break;
case "PowerOff":
$VirtualMachineStatus = $serviceVCenter->CheckVirtualMachine($product->vmware_id);
if ($VirtualMachineStatus === "Login_Again")
{
if ($VirtualMachineStatus === "Login_Again") {
$token = $serviceVCenter->loginServer($product->IpAddress->Host->Datacenter->vCenter);
if ($token)
{
vCenter::where('id',$product->IpAddress->Host->Datacenter->vCenter->id)->update(['token' => $token]);
if ($token) {
vCenter::where('id', $product->IpAddress->Host->Datacenter->vCenter->id)->update(['token' => $token]);
$serviceVCenter->token = $token;
}
$VirtualMachineStatus = $serviceVCenter->CheckVirtualMachine($product->vmware_id);
}
if ($VirtualMachineStatus['data']['operations_ready'] === false)
{
if ($VirtualMachineStatus['data']['operations_ready'] === false) {
$status = 0;
break;
}
$VirtualMachineStatus = $serviceVCenter->PowerOffVirtualMachine($product->vmware_id);
$jobs = new WaitStatusVirtualMachine($product->vmware_id,$serviceVCenter,$product->customer_id,true);
$status = 0;
break;
}
if ($VirtualMachineStatus && $VirtualMachineStatus['error'])
{
if ($VirtualMachineStatus && $VirtualMachineStatus['error']) {
throw new \Exception(json_encode($VirtualMachineStatus['data']));
}
if ($this->update)
{
$product->update(
['status' => $status]
);
if ($this->update) {
$product->update(['status' => $status]);
}
dispatch($jobs)->delay(now()->addSeconds(5));
}
public function failed(Throwable $exception)
{
Event(new NotificationStatus("Notification: " . $exception->getMessage(),false));
Event(new NotificationStatus("Notification: " . $exception->getMessage(), false));
$this->telegram->SendErrorMessage($exception->getMessage());
}
}
app/Jobs/VirtualMachine/Create/CreateVirtualMachineJob.php
+12
View File
@@ -7,6 +7,7 @@ namespace App\Jobs\VirtualMachine\Create;
use App\Http\Services\vCenter;
use App\Jobs\VirtualMachine\Provision\AssignIpToGuestJob;
use App\Models\OrderItem;
use App\Models\VmProvisionLock;
use App\Services\VirtualMachine\VmAddonResolver;
use App\Services\VirtualMachine\VmInfrastructureResolver;
use App\Services\VirtualMachine\VmIpReservationService;
@@ -55,8 +56,15 @@ class CreateVirtualMachineJob implements ShouldQueue
): bool {
$reservation = null;
$ipPoolId = null;
$lock = null;
try {
$customerId = $this->item->order->customer_id;
$lock = VmProvisionLock::updateOrCreate(
['order_item_id' => $this->item->id],
['customer_id' => $customerId, 'status' => 'running'],
);
$vmLogger->step('Create VM job started', [
'order_item_id' => $this->item->id,
'is_retry' => $this->isRetry,
@@ -198,12 +206,16 @@ class CreateVirtualMachineJob implements ShouldQueue
'vm_id' => $vmId,
]);
$lock?->update(['status' => 'success']);
return true;
} catch (Throwable $e) {
if ($reservation) {
$ipReservation->release($reservation, $ipPoolId);
}
$lock?->update(['status' => 'failed']);
$vmLogger->exception('Create VM job failed', $e, [
'order_item_id' => $this->item->id,
'is_retry' => $this->isRetry,
app/Jobs/VirtualMachine/Handle/WaitStatusVirtualMachine.php
+28 -47
View File
@@ -3,6 +3,8 @@
namespace App\Jobs\VirtualMachine\Handle;
use App\Events\VirtualMachine_Panel_Status;
use App\Http\Services\vCenter as ServiceVCenter;
use App\Models\ModuleVmware;
use Illuminate\Bus\Queueable;
use Illuminate\Console\Scheduling\Event;
use Illuminate\Contracts\Queue\ShouldQueue;
@@ -15,54 +17,49 @@ class WaitStatusVirtualMachine implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected string $vmware_id;
protected $serviceVCetner;
protected int $productId;
protected bool $status;
protected int $user;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct($vmware_id,$service,$user,$status)
public function __construct(int $productId, bool $status)
{
//
$this->vmware_id = $vmware_id;
$this->serviceVCetner = $service;
$this->user = $user;
$this->productId = $productId;
$this->status = $status;
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
public function handle(ServiceVCenter $serviceVCenter)
{
Log::info("WaitStatusVirtualMachine: Started for VM {$this->vmware_id}, Target Status: " . ($this->status ? 'True' : 'False'));
$product = ModuleVmware::with(['IpAddress.Host.Datacenter.vCenter'])->find($this->productId);
$status = $this->serviceVCetner->CheckVirtualMachine($this->vmware_id);
if (!$product || !$product->IpAddress?->Host?->Datacenter?->vCenter) {
Log::error("WaitStatusVirtualMachine: Product {$this->productId} missing vCenter relationship");
return;
}
$vcenter = $product->IpAddress->Host->Datacenter->vCenter;
$serviceVCenter->setConnection($vcenter->host, $vcenter->token);
$vmwareId = $product->vmware_id;
$userId = $product->customer_id;
Log::info("WaitStatusVirtualMachine: Started for VM {$vmwareId}, Target Status: " . ($this->status ? 'True' : 'False'));
$status = $serviceVCenter->CheckVirtualMachine($vmwareId);
if ($status['error'] === true) {
Log::error("WaitStatusVirtualMachine: Error checking status for VM {$this->vmware_id}: " . json_encode($status['message']));
Log::error("WaitStatusVirtualMachine: Error checking status for VM {$vmwareId}: " . json_encode($status['message']));
throw new \Exception(json_encode($status['message']));
}
Log::info("WaitStatusVirtualMachine: Initial operations_ready: " . ($status['data']['operations_ready'] ? 'True' : 'False'));
// Wait loop: "Wait while operations_ready matches $this->status"
// Example: Stop Job passes true. We wait while it is true. When it becomes false (Off), we exit.
if ($status['data']['operations_ready'] === $this->status) {
Log::info("WaitStatusVirtualMachine: Entering wait loop...");
do {
sleep(10);
$status = $this->serviceVCetner->CheckVirtualMachine($this->vmware_id);
$status = $serviceVCenter->CheckVirtualMachine($vmwareId);
if ($status['error'] === true) {
Log::warning("WaitStatusVirtualMachine: Failed to check status in loop. Breaking.");
// If check fails, maybe network glitch? We can break or continue.
// Let's break to avoid infinite loop on error
break;
}
@@ -73,17 +70,11 @@ class WaitStatusVirtualMachine implements ShouldQueue
Log::info("WaitStatusVirtualMachine: Skipping wait loop, condition already met.");
}
// Action Completed
// 1. Clear Cache again (to ensure next fetch gets new state)
\Illuminate\Support\Facades\Cache::forget(\App\Support\Cache\CacheKey::build('vm', 'info', ['id' => $this->vmware_id]));
\Illuminate\Support\Facades\Cache::forget(\App\Support\Cache\CacheKey::build('vm', 'info', ['id' => $vmwareId]));
// 2. Update DB Status and Dispatch Event
$product = \App\Models\ModuleVmware::where('vmware_id', $this->vmware_id)->first();
$product->refresh();
if ($product) {
Log::info("WaitStatusVirtualMachine: Product found. Updating status...");
// If waiting for True (On) to become False (Off) -> New Status 0 (Off)
// If waiting for False (Off) to become True (On) -> New Status 1 (Active)
$newStatus = ($this->status === true) ? 0 : 1;
Log::info("WaitStatusVirtualMachine: Setting status to $newStatus");
@@ -96,19 +87,9 @@ class WaitStatusVirtualMachine implements ShouldQueue
$statusStr = ($newStatus == 1) ? 'running' : 'stopped';
Log::info("WaitStatusVirtualMachine: Dispatching VirtualMachineStatusChanged event.");
\App\Events\VirtualMachineStatusChanged::dispatch(
$this->user,
$this->vmware_id,
$statusStr,
$msg
);
} else {
Log::error("WaitStatusVirtualMachine: Product not found for vmware_id: {$this->vmware_id}");
\App\Events\VirtualMachineStatusChanged::dispatch($userId, $vmwareId, $statusStr, $msg);
}
// Legacy event (optional, keep to avoid breaking other things if they listen to it)
Event(new VirtualMachine_Panel_Status('Yêu cầu được xử lý thành công', $this->user));
Event(new VirtualMachine_Panel_Status('Yêu cầu được xử lý thành công', $userId));
}
}
app/Jobs/VirtualMachine/Power/RebootVmJob.php
+1 -6
View File
@@ -71,12 +71,7 @@ class RebootVmJob implements ShouldQueue
// In VMJobs: $jobs = new WaitStatusVirtualMachine($product->vmware_id,$serviceVCenter,$product->customer_id,false);
dispatch(new WaitStatusVirtualMachine(
$product->vmware_id,
$vCenterService,
$this->userId,
false
))->delay(now()->addSeconds(5));
dispatch(new WaitStatusVirtualMachine($this->productId, false))->delay(now()->addSeconds(5));
} else {
$msg = is_array($result) ? ($result['data'] ?? 'Unknown error') : $result;
app/Jobs/VirtualMachine/Power/StartVmJob.php
+1 -6
View File
@@ -81,12 +81,7 @@ class StartVmJob implements ShouldQueue
// For Start:
// $jobs = new WaitStatusVirtualMachine($product->vmware_id, $serviceVCenter, $product->customer_id, false);
dispatch(new WaitStatusVirtualMachine(
$product->vmware_id,
$vCenterService,
$this->userId,
false // Expect operations_ready = false initially? Logic in WaitStatus is weird.
))->delay(now()->addSeconds(5));
dispatch(new WaitStatusVirtualMachine($this->productId, false))->delay(now()->addSeconds(5));
} else {
$msg = is_array($result) ? ($result['data'] ?? 'Unknown error') : $result;
app/Jobs/VirtualMachine/Power/StopVmJob.php
+1 -6
View File
@@ -69,12 +69,7 @@ class StopVmJob implements ShouldQueue
$product->update(['status' => 3]); // Processing
dispatch(new WaitStatusVirtualMachine(
$product->vmware_id,
$vCenterService,
$this->userId,
true
))->delay(now()->addSeconds(5));
dispatch(new WaitStatusVirtualMachine($this->productId, true))->delay(now()->addSeconds(5));
} else {
$msg = is_array($result) ? ($result['data'] ?? 'Unknown error') : $result;
app/Models/VmProvisionLock.php
+15
View File
@@ -0,0 +1,15 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Model;
class VmProvisionLock extends Model
{
protected $fillable = ['customer_id', 'order_item_id', 'status'];
public static function isRunning(int $customerId): bool
{
return static::where('customer_id', $customerId)->where('status', 'running')->exists();
}
}
app/Services/Api/Order/OrderService.php
+9
View File
@@ -6,6 +6,7 @@ use App\Enums\Order\OrderStatus;
use App\Helpers\OrderHelper;
use App\Jobs\Mail\Admin\AdminOrderNotificationJob;
use App\Jobs\VirtualMachine\Create\CreateVirtualMachineJob;
use App\Models\VmProvisionLock;
use App\Models\Addon;
use App\Models\Log as CustomerActivityLog;
use App\Models\Order;
@@ -224,6 +225,14 @@ class OrderService
private function dispatchVmJob(Order $order, OrderItem $item, Service $service, $customer): void
{
try {
if (VmProvisionLock::isRunning($customer->id)) {
Log::warning('dispatchVmJob: blocked — customer already has a running VM provision', [
'customer_id' => $customer->id,
'order_item_id' => $item->id,
]);
return;
}
dispatch(new CreateVirtualMachineJob($item));
Log::info('dispatchVmJob: CreateVirtualMachineJob dispatched', [
app/Services/VirtualMachine/VmIpReservationService.php
+2
View File
@@ -24,7 +24,9 @@ class VmIpReservationService
public function reserve(int $networkId, int $hostId, string $dns): ?array
{
return DB::transaction(function () use ($networkId, $hostId, $dns) {
// ponytail: exclude only IPs with an actual VM assigned, not stale __pending__ reservations
$usedIps = ModuleIpAddress::where('module_network_id', $networkId)
->where('vm_name', '!=', '__pending__')
->lockForUpdate()
->pluck('ip')
->toArray();
database/migrations/2026_07_13_000001_create_vm_provision_locks_table.php
+24
View File
@@ -0,0 +1,24 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::create('vm_provision_locks', function (Blueprint $table) {
$table->id();
$table->unsignedBigInteger('customer_id')->index();
$table->unsignedBigInteger('order_item_id')->unique();
$table->enum('status', ['running', 'success', 'failed'])->default('running');
$table->timestamps();
});
}
public function down(): void
{
Schema::dropIfExists('vm_provision_locks');
}
};