bao/portal-api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor VM provisioning pipeline into focused services and add IP pool

Browse Source 
management
This commit is contained in:
baodev0
2026-05-29 12:03:35 +07:00
parent 6bb4a47bfb
commit 7ae237ee9d
32 changed files with 1170 additions and 1200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Jobs/VirtualMachine/Create/CreateVirtualMachineJob.php
+126 -521
View File
@@ -1,570 +1,175 @@
<?php
declare(strict_types=1);
namespace App\Jobs\VirtualMachine\Create;
use App\Http\Services\Telegram;
use App\Http\Services\vCenter;
use App\Jobs\Mail\Customer\VMCreationFailedJob;
use App\Jobs\VirtualMachine\Provision\ChangePasswordViaVsphereJob;
use App\Services\VmProvisionService;
use App\Models\ModuleHost;
use App\Jobs\VirtualMachine\Provision\AssignIpToGuestJob;
use App\Models\OrderItem;
use App\Models\Os;
use App\Models\ModuleVmware;
use App\Models\Log as CustomerLog;
use App\Models\Service;
use App\Services\VirtualMachine\VmAddonResolver;
use App\Services\VirtualMachine\VmInfrastructureResolver;
use App\Services\VirtualMachine\VmModuleService;
use App\Services\VirtualMachine\VmNotificationService;
use App\Services\VirtualMachine\VmPowerService;
use App\Services\VirtualMachine\VmProvisionLogger;
use App\Services\VirtualMachine\VmServiceSyncService;
use App\Services\VirtualMachine\VmVcenterErrorTranslator;
use App\Services\VmProvisionService;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Http;
use RuntimeException;
use Throwable;
class CreateVirtualMachineJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
/** Cho phép job chạy tối đa 10 phút (VM clone có thể mất 5-8 phút) */
/** VM clone can take 5–8 minutes */
public int $timeout = 600;
/** Không retry tự động — lỗi VM cần admin kiểm tra thủ công */
/** No auto-retry — VM errors need manual review */
public int $tries = 1;
public OrderItem $item;
public bool $isRetry = false;
/**
* Create a new job instance.
*/
public function __construct(OrderItem $item, bool $isRetry = false)
{
$this->item = $item;
$this->isRetry = $isRetry;
public function __construct(
public OrderItem $item,
public bool $isRetry = false,
) {
}
/**
* Execute the job.
*/
public function handle(vCenter $vCenter, Telegram $telegram)
{
public function handle(
vCenter $vCenter,
VmProvisionLogger $vmLogger,
VmInfrastructureResolver $infraResolver,
VmAddonResolver $addonResolver,
VmPowerService $powerService,
VmModuleService $moduleService,
VmServiceSyncService $serviceSync,
VmNotificationService $notification,
VmVcenterErrorTranslator $errorTranslator,
): bool {
try {
$vmLogger->step('Create VM job started', [
'order_item_id' => $this->item->id,
'is_retry' => $this->isRetry,
]);
$product = $this->item->getProductAttribute();
if (!$product) {
$errorMessage = "Không tìm thấy sản phẩm từ OrderItem #{$this->item->id}";
Log::warning($errorMessage);
$telegram->SendErrorMessage($errorMessage, 'CreateVirtualMachineJob - Missing Product');
if (!$this->isRetry) {
$this->sendFailureNotification($errorMessage);
$this->writeCustomerFailureLog($errorMessage);
}
return false;
throw new RuntimeException("Không tìm thấy sản phẩm từ OrderItem #{$this->item->id}");
}
$osModel = Os::with(['host.datacenter.vCenter', 'osGroup.host.datacenter.vCenter'])->find($this->item->os_id);
$infra = $infraResolver->resolve($this->item, $product);
$addonData = $addonResolver->resolve($this->item);
if (!$osModel) {
$errorMessage = "Không tìm thấy OS id={$this->item->os_id} trong OrderItem #{$this->item->id}. os_id không hợp lệ.";
Log::error($errorMessage);
$telegram->SendErrorMessage([
'error' => $errorMessage,
'os_id' => $this->item->os_id,
'order_item_id' => $this->item->id,
'product_id' => $product->id,
], 'CreateVirtualMachineJob - Invalid OS ID');
if (!$this->isRetry) {
$this->sendFailureNotification($errorMessage);
$this->writeCustomerFailureLog($errorMessage);
}
return;
if (!$vCenter->loginServer($infra->vcenter)) {
throw new RuntimeException("Login vào vCenter thất bại với host: {$infra->vcenter->host}");
}
$template = $osModel->template_id;
// Ưu tiên 1: host gán trực tiếp trên OS
$host = $osModel->host;
$datacenter = $host?->datacenter;
$vcenter = $datacenter?->vCenter;
// Ưu tiên 2: host từ osGroup (cũ)
if (!$host || !$datacenter || !$vcenter) {
$groupHost = $osModel->osGroup?->host;
$host = $host ?? $groupHost;
$datacenter = $datacenter ?? $groupHost?->datacenter;
$vcenter = $vcenter ?? $groupHost?->datacenter?->vCenter;
if ($groupHost) {
Log::info('CreateVirtualMachineJob: dùng host từ osGroup làm fallback', [
'os_id' => $osModel->id,
'os_group_id' => $osModel->os_group_id,
'host_id' => $groupHost->id,
]);
}
}
// Ưu tiên 3: host từ sản phẩm (cấu hình tab Provisioning)
if ((!$host || !$datacenter || !$vcenter) && $product->module_host_id) {
$productHost = ModuleHost::with('datacenter.vCenter')->find($product->module_host_id);
if ($productHost) {
$host = $host ?? $productHost;
$datacenter = $datacenter ?? $productHost->datacenter;
$vcenter = $vcenter ?? $productHost->datacenter?->vCenter;
Log::info('CreateVirtualMachineJob: dùng host từ sản phẩm làm fallback', [
'product_id' => $product->id,
'host_id' => $productHost->id,
'datacenter_id' => $productHost->datacenter?->id,
]);
}
}
if (!$vcenter || !$host || !$datacenter) {
$errorMessage = "Thiếu thông tin host/datacenter/vcenter từ sản phẩm ID #{$product->id}. "
. "Vui lòng cấu hình vCenter trong tab Provisioning của sản phẩm, "
. "hoặc gán host cho OS group của OS được chọn.";
Log::error($errorMessage);
$telegram->SendErrorMessage([
'error' => $errorMessage,
'product_id' => $product->id,
'order_item_id' => $this->item->id,
'os_id' => $this->item->os_id,
'os_group_name' => $osModel->osGroup?->name,
'product_module_host_id' => $product->module_host_id,
'host_id' => $host?->id,
'datacenter_id' => $datacenter?->id,
'vcenter_id' => $vcenter?->id,
], 'CreateVirtualMachineJob - Missing Infrastructure');
if (!$this->isRetry) {
$this->sendFailureNotification($errorMessage);
$this->writeCustomerFailureLog($errorMessage);
}
return;
}
// Login to vCenter
if (!$vCenter->loginServer($vcenter)) {
$errorMessage = "Login vào vCenter thất bại với host: {$vcenter->host}";
Log::error($errorMessage);
$telegram->SendErrorMessage([
'error' => $errorMessage,
'vcenter_host' => $vcenter->host,
'vcenter_user' => $vcenter->user,
'product_id' => $product->id,
'order_item_id' => $this->item->id
], 'CreateVirtualMachineJob - vCenter Login Failed');
if (!$this->isRetry) {
$this->sendFailureNotification($errorMessage);
$this->writeCustomerFailureLog($errorMessage);
}
return false;
}
// Lấy thông tin addon (CPU, RAM)
$addonData = [
'cpu' => 0,
'ram' => 0,
'disk' => 0
];
// Lấy addon từ order item
if ($this->item->addons) {
foreach ($this->item->addons as $orderAddon) {
$addon = $orderAddon->addonItems;
if (!$addon)
continue;
$addonName = strtolower(trim($addon->name));
$quantity = (int) $orderAddon->qty;
switch ($addonName) {
case 'cpu':
$addonData['cpu'] += $quantity;
break;
case 'ram':
$addonData['ram'] += $quantity;
break;
case 'disk':
case 'ssd':
$addonData['disk'] += $quantity;
break;
}
}
}
// Xác định OS family để chọn username và luồng cấu hình phù hợp
$osGroupName = $osModel->osGroup?->name ?? '';
$isWindows = str_contains(strtolower($osGroupName), 'windows');
$vmUser = $isWindows ? 'administrator' : 'root';
$vmPort = $isWindows ? 3389 : 22;
$vmLogger->step('VM clone started', [
'order_item_id' => $this->item->id,
'product_id' => $product->id,
'template' => $infra->template,
'host_id' => $infra->host->id,
'vcenter_id' => $infra->vcenter->id,
]);
$result = $vCenter->CreateVirtualMachine([
'item' => $this->item,
'host' => $host,
'datacenter' => $datacenter,
'vcenter' => $vcenter,
'template' => $template,
'addon' => $addonData,
'item' => $this->item,
'host' => $infra->host,
'datacenter' => $infra->datacenter,
'vcenter' => $infra->vcenter,
'template' => $infra->template,
'addon' => $addonData,
]);
Log::info('CreateVirtualMachineJob - Result: ' . json_encode($result));
if (is_array($result) && ($result['error'] ?? false)) {
$errorMessage = $this->translateVcenterError($result);
Log::warning($errorMessage);
$telegram->SendErrorMessage([
'error' => $result['error'] ?? 'Unknown error',
'result' => $result,
'product_id' => $product->id,
'order_item_id' => $this->item->id,
'host_name' => $host?->name,
'datacenter_name' => $datacenter?->name,
'vcenter_host' => $vcenter?->host
], 'CreateVirtualMachineJob - VM Creation Failed');
if (!$this->isRetry) {
$this->sendFailureNotification($errorMessage);
$this->writeCustomerFailureLog($errorMessage);
}
return false;
} elseif ($result === 'Login_Again') {
$errorMessage = 'Tạo máy ảo thất bại: cần đăng nhập lại.';
Log::error($errorMessage);
$telegram->SendErrorMessage([
'error' => $errorMessage,
'product_id' => $product->id,
'order_item_id' => $this->item->id,
'vcenter_host' => $vcenter?->host
], 'CreateVirtualMachineJob - vCenter Re-login Required');
if (!$this->isRetry) {
$this->sendFailureNotification($errorMessage);
$this->writeCustomerFailureLog($errorMessage);
}
return false;
} else {
Log::info('Tạo máy ảo thành công: ' . json_encode($result));
// Lấy vm_id từ kết quả
$vmId = is_array($result) ? ($result['data'] ?? 'unknown') : $result;
if ($vmId !== 'unknown' && $vmId !== 'UNKNOWN') {
try {
// vm_id được lưu vào tbl.services qua ModuleVmwareObserver::created()
$startResult = $this->startVirtualMachineDirectly($vCenter, $vmId);
// VM started — SSH vào vSphere host để đổi mật khẩu qua PS1 script
try {
$customer = $this->item->order->customer;
$serverName = $this->item->server_name ?? 'VM-' . $vmId;
$months = (int) ($this->item->months ?? 0);
$firstPayment = (int) $this->item->price * ($months ?: 1);
$nextDueDate = $months > 0 ? now()->addMonths($months)->timestamp : null;
// Create ModuleVmware without IP — AssignIpToGuestJob will
// pick, assign, and verify the real IP later, then write it to DB.
$moduleVmware = ModuleVmware::create([
'name' => $serverName,
'module_ip_address' => null, // set by AssignIpToGuestJob
'ip' => null, // set by AssignIpToGuestJob
'port' => $vmPort,
'order_id' => $this->item->order->id,
'first_payment' => $firstPayment,
'next_payment' => $firstPayment,
'next_due_date' => $nextDueDate,
'next_invoice' => $nextDueDate,
'user' => $vmUser,
'password' => VmProvisionService::TEMPLATE_PASSWORD,
'plain_password' => VmProvisionService::TEMPLATE_PASSWORD,
'hostname' => $serverName,
'vmware_id' => $vmId,
'vcenter_id' => $vcenter->id,
'os_id' => $this->item->os_id ?? 1,
'customer_id' => $customer->id,
'status' => 1,
]);
// Sync services table
$this->syncServiceRecord($moduleVmware);
// Step 1 of guest provisioning: change template password.
// On success ChangePasswordViaVsphereJob dispatches AssignIpToGuestJob,
// which handles IP selection, assignment, verification, and email.
ChangePasswordViaVsphereJob::dispatch(
$moduleVmware->id,
$vmId,
$isWindows ? VmProvisionService::OS_WINDOWS : VmProvisionService::OS_LINUX,
);
Log::info('ChangePasswordViaVsphereJob dispatched', [
'module_vmware_id' => $moduleVmware->id,
'vm_id' => $vmId,
'os_family' => $isWindows ? 'WINDOWS' : 'LINUX',
]);
} catch (\Exception $e) {
Log::error('❌ Lỗi khi lưu dữ liệu VM và IP address: ' . $e->getMessage(), [
'vmware_id' => $vmId,
'order_item_id' => $this->item->id,
'error' => $e->getMessage()
]);
}
} catch (\Exception $e) {
Log::error('❌ Lỗi khi lưu dữ liệu VM hoặc start VM: ' . $e->getMessage(), [
'vm_id' => $vmId,
'order_id' => $this->item->order->id,
'trace' => $e->getTraceAsString()
]);
}
}
$telegram->SendJobRunInfo(
'CreateVirtualMachineJob',
'success',
[
'product_id' => $product->id,
'order_item_id' => $this->item->id,
'result' => $result,
'addon_applied' => !empty($addonData) && array_sum($addonData) > 0,
'vm_id_saved' => $vmId !== 'unknown' && $vmId !== 'UNKNOWN',
'vm_started' => isset($startResult) && is_array($startResult) && $startResult['error'] === false
]
throw new RuntimeException(
$errorTranslator->translate($result, $this->item->server_name)
);
$this->writeCustomerSuccessLog();
try {
$order = $this->item->order;
$customer = $order->customer;
broadcast(new \App\Events\OrderProvisioned(
orderId: $order->id,
orderNum: $order->order_num,
customerName: trim($customer->firstname . ' ' . $customer->lastname),
vmId: (string) $vmId,
success: true,
message: 'Máy ảo đã được khởi tạo thành công cho khách hàng ' . trim($customer->firstname . ' ' . $customer->lastname) . ' (Đơn #' . $order->order_num . ')',
));
} catch (\Throwable $e) {
Log::warning('Không thể broadcast OrderProvisioned: ' . $e->getMessage());
}
return true;
}
} catch (\Throwable $e) {
$errorMessage = 'Lỗi khi tạo máy ảo (Job): ' . $e->getMessage();
Log::error($errorMessage, [
'exception' => $e,
if ($result === 'Login_Again') {
throw new RuntimeException('Tạo máy ảo thất bại: cần đăng nhập lại vCenter.');
}
$vmId = is_array($result) ? ($result['data'] ?? null) : $result;
if (!$vmId || in_array($vmId, ['unknown', 'UNKNOWN'], true)) {
throw new RuntimeException('Không lấy được vm_id sau khi tạo máy ảo.');
}
$vmLogger->success('VM clone success', [
'order_item_id' => $this->item->id,
'trace' => $e->getTraceAsString()
'vm_id' => $vmId,
]);
$telegram->SendErrorMessage([
'error' => $e->getMessage(),
'file' => $e->getFile(),
'line' => $e->getLine(),
$vmLogger->step('VM start started', ['vm_id' => $vmId]);
$startResult = $powerService->start($vCenter, $vmId);
if (is_array($startResult) && ($startResult['error'] ?? false)) {
throw new RuntimeException(
'Start VM thất bại: ' . json_encode($startResult['data'] ?? null)
);
}
if ($startResult === 'Login_Again') {
throw new RuntimeException('Start VM thất bại: cần đăng nhập lại vCenter.');
}
$vmLogger->success('VM start success', ['vm_id' => $vmId]);
$moduleVmware = $moduleService->create($this->item, $infra, $vmId);
$vmLogger->success('ModuleVmware created', [
'module_vmware_id' => $moduleVmware->id,
'vm_id' => $vmId,
]);
$serviceSync->sync($moduleVmware, $this->item);
$vmLogger->success('Service synced', [
'module_vmware_id' => $moduleVmware->id,
'order_item_id' => $this->item->id,
]);
AssignIpToGuestJob::dispatch(
$moduleVmware->id,
(string) $vmId,
$infra->isWindows ? VmProvisionService::OS_WINDOWS : VmProvisionService::OS_LINUX,
VmProvisionService::TEMPLATE_PASSWORD,
)->onQueue('default');
$vmLogger->success('AssignIpToGuestJob dispatched', [
'module_vmware_id' => $moduleVmware->id,
'vm_id' => $vmId,
'os_family' => $infra->isWindows ? 'WINDOWS' : 'LINUX',
]);
$notification->success($this->item, $moduleVmware, (string) $vmId);
$vmLogger->success('Create VM job success', [
'order_item_id' => $this->item->id,
'module_vmware_id' => $moduleVmware->id,
'vm_id' => $vmId,
]);
return true;
} catch (Throwable $e) {
$vmLogger->exception('Create VM job failed', $e, [
'order_item_id' => $this->item->id,
'trace' => substr($e->getTraceAsString(), 0, 500) // Giới hạn độ dài trace
], 'CreateVirtualMachineJob - Exception');
'is_retry' => $this->isRetry,
]);
if (!$this->isRetry) {
$this->sendFailureNotification($errorMessage);
$this->writeCustomerFailureLog($errorMessage);
$notification->failed($this->item, $e->getMessage());
}
return false;
}
}
private function writeCustomerSuccessLog(): void
{
try {
$order = $this->item->order;
$customer = $order?->customer;
if (!$order || !$customer) {
return;
}
CustomerLog::create([
'agent' => 'system',
'client_ip' => null,
'customer_id' => $customer->id,
'description' => 'Khởi tạo dịch vụ thành công cho đơn hàng #' . $order->order_num
. ' (product_id: ' . $this->item->product_id . ', os_id: ' . $this->item->os_id . ').',
]);
} catch (\Throwable $e) {
Log::warning('Không thể ghi log thành công khởi tạo VM: ' . $e->getMessage());
}
}
private function writeCustomerFailureLog(string $errorMessage): void
{
try {
$order = $this->item->order;
$customer = $order?->customer;
if (!$order || !$customer) {
return;
}
CustomerLog::create([
'agent' => 'system',
'client_ip' => null,
'customer_id' => $customer->id,
'description' => 'Khởi tạo dịch vụ thất bại cho đơn hàng #' . $order->order_num
. ' (product_id: ' . $this->item->product_id . ', os_id: ' . $this->item->os_id . ').'
. ' Lỗi: ' . $errorMessage . '. '
. 'Nếu có lỗi xảy ra khi khởi tạo dịch vụ quý khách nhấn khởi tạo tại đây, hoặc quý khách có thể liên hệ với kỹ thuật support@tgs.com.vn',
]);
} catch (\Throwable $e) {
Log::warning('Không thể ghi log thất bại khởi tạo VM: ' . $e->getMessage());
}
}
/**
* Start VM directly using vCenter API
*/
private function startVirtualMachineDirectly(vCenter $vCenter, string $vmId): array|string
{
try {
// Kiểm tra xem đã có session chưa
$check = $vCenter->checkApiExits();
if (!$check) {
return 'Login_Again';
}
// Gọi API start VM trực tiếp
$response = Http::withoutVerifying()
->withHeaders([
'vmware-api-session-id' => $vCenter->token
])
->post("https://{$vCenter->apiUri}/rest/vcenter/vm/{$vmId}/power/start");
if ($response->status() === 200) {
return [
'error' => false,
'data' => 'success'
];
}
return [
'error' => true,
'data' => $response->body()
];
} catch (\Exception $e) {
Log::error('Lỗi khi start VM: ' . $e->getMessage());
return [
'error' => true,
'data' => $e->getMessage()
];
}
}
/**
* Send failure notification to customer
*/
private function sendFailureNotification(string $errorMessage): void
{
try {
$customer = $this->item->order->customer;
$product = $this->item->getProductAttribute();
VMCreationFailedJob::dispatch([
'customer_email' => $customer->email,
'customer_name' => trim($customer->firstname . ' ' . $customer->lastname),
'product_name' => $product?->name ?? 'Unknown Product',
'order_id' => $this->item->order->id,
'order_num' => $this->item->order->order_num,
'error_message' => $errorMessage,
'order_item_id' => $this->item->id
]);
} catch (\Throwable $e) {
Log::error('Không thể gửi email thông báo lỗi VM: ' . $e->getMessage());
}
}
/**
* Sync module_vmware info → services table.
* Link: module_vmware.order_id → order_items.order_id → services.order_item_id
*/
private function syncServiceRecord(ModuleVmware $moduleVmware): void
{
try {
$orderItem = OrderItem::where('order_id', $moduleVmware->order_id)->first();
if (!$orderItem) {
Log::warning('syncServiceRecord: không tìm thấy OrderItem', [
'order_id' => $moduleVmware->order_id,
'module_vmware_id' => $moduleVmware->id,
]);
return;
}
$months = (int) ($orderItem->months ?? 0);
$updated = Service::where('order_item_id', $orderItem->id)
->update([
'vm_id' => $moduleVmware->id,
'status' => 'active',
'expires_at' => $months > 0 ? now()->addMonths($months) : null,
]);
Log::info('syncServiceRecord: service synced', [
'module_vmware_id' => $moduleVmware->id,
'vmware_id' => $moduleVmware->vmware_id,
'order_item_id' => $orderItem->id,
'rows_updated' => $updated,
]);
} catch (\Throwable $e) {
Log::error('syncServiceRecord: exception — ' . $e->getMessage(), [
'module_vmware_id' => $moduleVmware->id,
'order_id' => $moduleVmware->order_id,
]);
}
}
/**
* Gửi email thông báo VM đã khởi tạo thành công cho khách hàng.
* Gọi ngay sau khi tạo module_vmware — cho cả Linux lẫn Windows.
*/
/**
* Dịch error từ vCenter API sang tiếng Việt dễ hiểu.
*/
private function translateVcenterError(array $result): string
{
$data = $result['data'] ?? [];
$errorType = is_array($data) ? ($data['error_type'] ?? '') : '';
$serverName = $this->item->server_name ?? '';
$map = [
'ALREADY_EXISTS' => "Tên máy chủ \"{$serverName}\" đã tồn tại trong hệ thống. Vui lòng liên hệ hỗ trợ để được xử lý hoặc chọn tên máy chủ khác.",
'NOT_FOUND' => 'Không tìm thấy template hoặc tài nguyên trên vCenter. Vui lòng liên hệ hỗ trợ.',
'RESOURCE_INACCESSIBLE' => 'Tài nguyên vCenter không khả dụng. Vui lòng thử lại sau hoặc liên hệ hỗ trợ.',
'INVALID_ARGUMENT' => 'Thông tin cấu hình máy chủ không hợp lệ. Vui lòng liên hệ hỗ trợ.',
'UNABLE_TO_ALLOCATE_RESOURCE' => 'Không đủ tài nguyên để khởi tạo máy chủ. Vui lòng liên hệ hỗ trợ.',
'TIMED_OUT' => 'Yêu cầu khởi tạo máy chủ hết thời gian chờ. Vui lòng thử lại sau.',
];
if ($errorType && isset($map[$errorType])) {
return $map[$errorType];
}
// Fallback: không để lộ JSON thô ra email khách hàng
return "Không thể khởi tạo máy chủ ảo lúc này. Chúng tôi đã ghi nhận và sẽ xử lý trong thời gian sớm nhất. Vui lòng liên hệ hỗ trợ nếu cần.";
}
private function getBillingCycleLabel(?int $months): string
{
return match ($months) {
1 => 'Hàng tháng',
3 => '3 Tháng',
6 => '6 Tháng',
12 => 'Hàng năm',
24 => '2 Năm',
default => $months . ' Tháng',
};
}
}
app/Jobs/VirtualMachine/Provision/AssignIpToGuestJob.php
+109 -42
View File
@@ -9,8 +9,11 @@ use App\Models\IpPool;
use App\Models\ModuleIpAddress;
use App\Models\ModuleVmware;
use App\Models\Os;
use App\Models\Service;
use App\Models\VirtualMachines;
use App\Http\Services\vCenter as vCenterService;
use App\Models\vCenter;
use App\Services\VmProvisionService;
use App\Services\VsphereHostSshService;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
@@ -24,7 +27,8 @@ use Illuminate\Support\Facades\Log;
* AssignIpToGuestJob
*
* Assigns a real static IP to the VM guest OS and verifies it is active.
* Only persists the IP to DB after successful guest-side verification.
* Only persists the IP to DB (module_vmware + virtual_machines) after
* successful guest-side verification, then dispatches ChangePasswordViaVsphereJob.
*
* Flow:
* [1] Resolve network config from order item → product → ipAddress template
@@ -34,7 +38,7 @@ use Illuminate\Support\Facades\Log;
* [5] On FAIL → release IP, try next
* [6] Call verifyIPLinux.ps1 / verifyIPWindows.ps1 via SSH
* [7] On FAIL → release IP, try next
* [8] SUCCESS → persistIp() → dispatch SendVmCredentialsEmailJob
* [8] SUCCESS → persistIp() → dispatch ChangePasswordViaVsphereJob
* [9] All attempts exhausted → mark VM inactive (status=0)
*/
class AssignIpToGuestJob implements ShouldQueue
@@ -52,8 +56,8 @@ class AssignIpToGuestJob implements ShouldQueue
public function __construct(
public readonly int $moduleVmwareId,
public readonly string $vmId,
public readonly string $osFamily, // VmProvisionService::OS_LINUX | OS_WINDOWS
public readonly string $guestPassword, // New password from ChangePasswordViaVsphereJob
public readonly string $osFamily, // VmProvisionService::OS_LINUX | OS_WINDOWS
public readonly string $guestPassword = VmProvisionService::TEMPLATE_PASSWORD,
) {}
// ─────────────────────────────────────────────────────────────────────────
@@ -79,16 +83,15 @@ class AssignIpToGuestJob implements ShouldQueue
return;
}
// Resolve network config: product template IP → gateway, netmask, dns, networkId
// DNS + network selector still come from product template; gateway + CIDR
// are now read from the chosen ip_pools row at reservation time.
$orderItem = $moduleVmware->order?->orderItems?->first();
$osModel = $orderItem?->os_id ? Os::with('osGroup.host')->find($orderItem->os_id) : null;
$networkId = $orderItem?->product?->ipAddress?->module_network_id ?? 1;
$gateway = $orderItem?->product?->ipAddress?->gateway ?? '0.0.0.0';
$netmask = $orderItem?->product?->ipAddress?->netmask ?? '255.255.255.0';
$dns = $orderItem?->product?->ipAddress?->dns ?? '8.8.8.8';
$hostId = $osModel?->osGroup?->host?->id ?? 1;
$cidr = (string) $this->netmaskToCidr($netmask);
$dns = $orderItem?->product?->ipAddress?->dns ?? '8.8.8.8';
$hostId = $osModel?->osGroup?->host?->id ?? 1;
$serviceId = Service::where('vm_id', $this->moduleVmwareId)->value('id');
// vCenter credentials
$vcenter = $moduleVmware->vcenter ?? vCenter::first();
@@ -109,9 +112,9 @@ class AssignIpToGuestJob implements ShouldQueue
]);
// [3] Reserve a free IP with transaction + row lock
$reservation = $this->reserveIp($networkId, $hostId, $gateway, $netmask, $dns);
$picked = $this->reserveIp($networkId, $hostId, $dns);
if (!$reservation) {
if (!$picked) {
Log::error('AssignIpToGuestJob: no free IPs in pool', [
'network_id' => $networkId,
'attempt' => $attempt,
@@ -119,9 +122,13 @@ class AssignIpToGuestJob implements ShouldQueue
break; // No point retrying — no IPs available
}
$ip = $reservation->ip;
$reservation = $picked['reservation'];
$ipPoolId = $picked['ip_pool_id'];
$ip = $reservation->ip;
$gateway = (string) $picked['gateway'];
$cidr = (string) $picked['cidr'];
// [5] Assign IP into VM guest OS via PS1 script
// [4] Assign IP into VM guest OS via PS1 script (Linux uses assignIPLinux.ps1)
$assignOutput = $ssh->assignIp(
vmId: $this->vmId,
ip: $ip,
@@ -136,18 +143,18 @@ class AssignIpToGuestJob implements ShouldQueue
guestUser: $guestUser,
);
// [6] If assign failed → release and try next IP
// [5] If assign failed → release and try next IP
if (!$this->isSuccess($assignOutput)) {
Log::warning('AssignIpToGuestJob: assignIp FAIL', [
'ip' => $ip,
'attempt' => $attempt,
'output' => trim($assignOutput),
]);
$this->releaseIp($reservation);
$this->releaseIp($reservation, $ipPoolId);
continue;
}
// [7] Verify IP is active inside guest OS via PS1 script
// [6] Verify IP is active inside guest OS via PS1 script
$verifyOutput = $ssh->verifyIpInGuest(
vmId: $this->vmId,
ip: $ip,
@@ -159,19 +166,19 @@ class AssignIpToGuestJob implements ShouldQueue
guestUser: $guestUser,
);
// [8] If verification failed → release and try next IP
// [7] If verification failed → release and try next IP
if (!$this->isSuccess($verifyOutput)) {
Log::warning('AssignIpToGuestJob: verifyIpInGuest FAIL', [
'ip' => $ip,
'attempt' => $attempt,
'output' => trim($verifyOutput),
]);
$this->releaseIp($reservation);
$this->releaseIp($reservation, $ipPoolId);
continue;
}
// [9] SUCCESS — persist confirmed IP to DB
$this->persistIp($reservation, $moduleVmware);
// [8] SUCCESS — persist confirmed IP to module_vmware + virtual_machines + ip_pools
$this->persistIp($reservation, $moduleVmware, $ipPoolId, $serviceId);
Log::info('AssignIpToGuestJob: IP assigned and verified', [
'module_vmware_id' => $this->moduleVmwareId,
@@ -180,13 +187,17 @@ class AssignIpToGuestJob implements ShouldQueue
'attempt' => $attempt,
]);
// Dispatch credential email now that we have the real IP
SendVmCredentialsEmailJob::dispatch($this->moduleVmwareId);
// Dispatch password change now that IP is confirmed
ChangePasswordViaVsphereJob::dispatch(
$this->moduleVmwareId,
$this->vmId,
$this->osFamily,
);
return;
}
// [10] All attempts exhausted
// [9] All attempts exhausted
Log::error('AssignIpToGuestJob: all attempts failed — marking VM inactive', [
'module_vmware_id' => $this->moduleVmwareId,
'vm_id' => $this->vmId,
@@ -194,9 +205,10 @@ class AssignIpToGuestJob implements ShouldQueue
]);
ModuleVmware::where('id', $this->moduleVmwareId)->update(['status' => 0]);
Service::where('vm_id', $this->moduleVmwareId)->update(['status' => 'failed']);
// Still send credential email — try to get IP from vSphere guest identity
// so the customer at least gets server info even if IP assignment failed
// Attempt to recover IP from vSphere guest identity for admin visibility only.
// Do not send credentials here: password has not been rotated yet.
try {
$moduleVmware = ModuleVmware::find($this->moduleVmwareId);
if ($moduleVmware && !$moduleVmware->ip) {
@@ -215,13 +227,8 @@ class AssignIpToGuestJob implements ShouldQueue
}
}
}
SendVmCredentialsEmailJob::dispatch($this->moduleVmwareId);
Log::info('AssignIpToGuestJob: dispatched SendVmCredentialsEmailJob after failed attempts', [
'module_vmware_id' => $this->moduleVmwareId,
]);
} catch (\Throwable $e) {
Log::error('AssignIpToGuestJob: failed to send email after exhausted attempts', [
Log::error('AssignIpToGuestJob: failed to recover IP after exhausted attempts', [
'module_vmware_id' => $this->moduleVmwareId,
'error' => $e->getMessage(),
]);
@@ -237,10 +244,11 @@ class AssignIpToGuestJob implements ShouldQueue
]);
ModuleVmware::where('id', $this->moduleVmwareId)->update(['status' => 0]);
Service::where('vm_id', $this->moduleVmwareId)->update(['status' => 'failed']);
}
// ─────────────────────────────────────────────────────────────────────────
// assignIp / verifyIp helpers
// IP reservation helpers
// ─────────────────────────────────────────────────────────────────────────
/**
@@ -253,14 +261,15 @@ class AssignIpToGuestJob implements ShouldQueue
*
* Returns the pending ModuleIpAddress record, or null if pool is exhausted.
*/
/**
* @return array{reservation:ModuleIpAddress,ip_pool_id:int,gateway:string,cidr:int}|null
*/
private function reserveIp(
int $networkId,
int $hostId,
string $gateway,
string $netmask,
string $dns,
): ?ModuleIpAddress {
return DB::transaction(function () use ($networkId, $hostId, $gateway, $netmask, $dns) {
): ?array {
return DB::transaction(function () use ($networkId, $hostId, $dns) {
// Lock existing rows to prevent another job from picking the same IP
$usedIps = ModuleIpAddress::where('module_network_id', $networkId)
->lockForUpdate()
@@ -268,6 +277,9 @@ class AssignIpToGuestJob implements ShouldQueue
->toArray();
$ipPool = IpPool::where('module_network_id', $networkId)
->where('status', 'free')
->whereNotNull('gateway')
->where('gateway', '!=', '')
->whereNotIn('ip_address', $usedIps)
->lockForUpdate()
->inRandomOrder()
@@ -277,8 +289,15 @@ class AssignIpToGuestJob implements ShouldQueue
return null;
}
// Hold the pool slot
$ipPool->update(['status' => 'reserved']);
$cidr = (int) ($ipPool->cidr ?: 24);
$gateway = (string) ($ipPool->gateway ?? '');
$netmask = $this->cidrToNetmask($cidr);
// Create a pending reservation — acts as a distributed lock on this IP
return ModuleIpAddress::create([
$reservation = ModuleIpAddress::create([
'vm_name' => '__pending__',
'module_host_id' => $hostId,
'module_network_id' => $networkId,
@@ -288,6 +307,13 @@ class AssignIpToGuestJob implements ShouldQueue
'dns' => $dns,
'status' => 0, // 0 = pending/reserved, not yet confirmed
]);
return [
'reservation' => $reservation,
'ip_pool_id' => $ipPool->id,
'gateway' => $gateway,
'cidr' => $cidr,
];
});
}
@@ -295,10 +321,13 @@ class AssignIpToGuestJob implements ShouldQueue
* Release a failed reservation by deleting the pending ModuleIpAddress row.
* This returns the IP to the available pool.
*/
private function releaseIp(ModuleIpAddress $reservation): void
private function releaseIp(ModuleIpAddress $reservation, ?int $ipPoolId): void
{
try {
$reservation->delete();
if ($ipPoolId) {
IpPool::where('id', $ipPoolId)->update(['status' => 'free']);
}
Log::info('AssignIpToGuestJob: IP reservation released', ['ip' => $reservation->ip]);
} catch (\Throwable $e) {
Log::warning('AssignIpToGuestJob: could not release IP reservation', [
@@ -312,12 +341,17 @@ class AssignIpToGuestJob implements ShouldQueue
* Confirm a successful IP assignment:
* - Update ModuleIpAddress: status → 1 (assigned), vm_name → real name
* - Update ModuleVmware: ip, dns, module_ip_address FK
* - Update VirtualMachines: ip, ip_pool_id FK (matched by vmId)
*
* IP is only written to DB AFTER guest-side verification passes.
*/
private function persistIp(ModuleIpAddress $reservation, ModuleVmware $moduleVmware): void
{
DB::transaction(function () use ($reservation, $moduleVmware) {
private function persistIp(
ModuleIpAddress $reservation,
ModuleVmware $moduleVmware,
?int $ipPoolId,
?int $serviceId,
): void {
DB::transaction(function () use ($reservation, $moduleVmware, $ipPoolId, $serviceId) {
$reservation->update([
'vm_name' => $moduleVmware->name ?? ('VM-' . $this->vmId),
'status' => 1, // assigned
@@ -328,9 +362,38 @@ class AssignIpToGuestJob implements ShouldQueue
'dns' => $reservation->dns,
'module_ip_address' => $reservation->id,
]);
// Sync to virtual_machines — matched by vSphere vmId
VirtualMachines::where('vm_id', $this->vmId)->update([
'ip' => $reservation->ip,
'ip_pool_id' => $ipPoolId,
]);
// Flip ip_pools row to assigned and link service
if ($ipPoolId) {
IpPool::where('id', $ipPoolId)->update([
'status' => 'assigned',
'service_id' => $serviceId,
]);
}
// Touch services row so server_name reflects the provisioned VM.
// Note: services.vm_id is an FK to module_vmware.id (already set by VmServiceSyncService) — do not overwrite with vSphere vmId string.
if ($serviceId) {
Service::where('id', $serviceId)->update([
'server_name' => $moduleVmware->name,
]);
}
});
}
/** Convert CIDR prefix length to dotted-decimal netmask (e.g. 24 → 255.255.255.0). */
private function cidrToNetmask(int $cidr): string
{
$cidr = max(0, min(32, $cidr));
return long2ip(-1 << (32 - $cidr));
}
// ─────────────────────────────────────────────────────────────────────────
// Micro-helpers
// ─────────────────────────────────────────────────────────────────────────
@@ -338,6 +401,10 @@ class AssignIpToGuestJob implements ShouldQueue
/** PS1 scripts return JSON: {"status":"ok"} or {"status":"error","message":"..."} */
private function isSuccess(string $output): bool
{
if (str_contains(strtoupper(trim($output)), 'SUCCESS')) {
return true;
}
$data = json_decode(trim($output), true);
return is_array($data) && ($data['status'] ?? '') === 'ok';
}
app/Jobs/VirtualMachine/Provision/ChangePasswordViaVsphereJob.php
+10 -5
View File
@@ -6,6 +6,7 @@ namespace App\Jobs\VirtualMachine\Provision;
use App\Http\Facade\SignRSA;
use App\Models\ModuleVmware;
use App\Models\Service;
use App\Models\vCenter;
use App\Services\VmProvisionService;
use App\Services\VsphereHostSshService;
@@ -22,15 +23,15 @@ use Illuminate\Support\Facades\Log;
* Resets the VM guest OS password via:
* Laravel → SSH (10.10.10.173) → PowerShell PS1 → Invoke-VMScript (guest OS)
*
* Flow:
* [1] VM already exists, vm_id known
* Dispatched by AssignIpToGuestJob after IP is confirmed. Flow:
* [1] VM already exists, vm_id known, IP already assigned
* [2] Generate random 16-char password (plain text)
* [3] Load vCenter credentials from module_vcenter
* [4] SSH into vSphere host
* [5] Execute changePassword{Linux|Windows}.ps1 with 5 params
* [6] Parse "SUCCESS" / "FAIL" from script output
* [7] On SUCCESS → save encrypted password to DB
* [8] Dispatch AssignIpToGuestJob with the new guest password
* [8] Dispatch SendVmCredentialsEmailJob (IP + password both ready)
* [9] On FAIL → throw exception → job retries
*/
class ChangePasswordViaVsphereJob implements ShouldQueue
@@ -84,10 +85,13 @@ class ChangePasswordViaVsphereJob implements ShouldQueue
// [6] Parse output — throw on FAIL so job retries
$this->assertSuccess($output);
// [7] Save encrypted + plain password (IP will be set later by AssignIpToGuestJob)
// [7] Save encrypted + plain password (IP already assigned by AssignIpToGuestJob)
$provision->updatePassword($this->moduleVmwareId, $newPassword);
// [8] Send VM credentials email (AssignIpToGuestJob bypassed temporarily)
// Provisioning complete — activate the service record
Service::where('vm_id', $this->moduleVmwareId)->update(['status' => 'active']);
// [8] Send credentials email — IP + new password are both ready at this point
SendVmCredentialsEmailJob::dispatch($this->moduleVmwareId);
Log::info('ChangePasswordViaVsphereJob: completed', [
@@ -107,6 +111,7 @@ class ChangePasswordViaVsphereJob implements ShouldQueue
]);
ModuleVmware::where('id', $this->moduleVmwareId)->update(['status' => 0]);
Service::where('vm_id', $this->moduleVmwareId)->update(['status' => 'failed']);
}
// ─────────────────────────────────────────────────────────────────────────
app/Jobs/VirtualMachine/Provision/WaitVmReadyJob.php
-2
View File
@@ -102,9 +102,7 @@ class WaitVmReadyJob implements ShouldQueue
ChangePasswordViaVsphereJob::dispatch(
$this->moduleVmwareId,
$this->vmId,
$guestIp,
$this->osFamily,
$this->username,
);
}