for('AssignIpToGuestJob'); $logger->step('start', [ 'module_vmware_id' => $this->moduleVmwareId, 'vm_id' => $this->vmId, 'os_family' => $this->osFamily, ]); // Load ModuleVmware with the relations needed to derive network config $moduleVmware = ModuleVmware::with([ 'order.orderItems.product.ipAddress', 'vcenter', ])->find($this->moduleVmwareId); if (!$moduleVmware) { $logger->error('ModuleVmware not found', [ 'module_vmware_id' => $this->moduleVmwareId, ]); return; } // Network preferably comes from the OS's own binding (module_network_id); // falls back to the product's ipAddress template for legacy OS entries // that were never assigned a network. Gateway + CIDR are read from the // chosen ip_pools row at reservation time. $orderItem = $moduleVmware->order?->orderItems?->first(); $osModel = $orderItem?->os_id ? Os::with('osGroup.host')->find($orderItem->os_id) : null; $networkId = $osModel?->module_network_id ?? $orderItem?->product?->ipAddress?->module_network_id ?? 1; $dns = $orderItem?->product?->ipAddress?->dns ?? '8.8.8.8'; $hostId = $osModel?->osGroup?->host?->id ?? 1; $serviceId = Service::where('vm_id', $this->moduleVmwareId)->value('id'); // vCenter credentials $vcenter = $moduleVmware->vcenter ?? vCenter::first(); $vcenterHost = $vcenter->host; $vcenterUser = $vcenter->user; $vcenterPass = SignRSA::DecryptFunc($vcenter->password); // Guest OS default admin user per OS family $guestUser = strtoupper($this->osFamily) === 'WINDOWS' ? 'Administrator' : 'root'; // ─── Attempt loop ──────────────────────────────────────────────────── // If reservationId is supplied, the IP was reserved upstream (CreateVirtualMachineJob) // and baked into the VM name — we can't swap IPs on retry, so we run a single attempt. $maxAttempts = $this->reservationId ? 1 : self::MAX_ATTEMPTS; for ($attempt = 1; $attempt <= $maxAttempts; $attempt++) { $logger->step('attempt', [ 'attempt' => $attempt, 'module_vmware_id' => $this->moduleVmwareId, 'network_id' => $networkId, 'preserved' => (bool) $this->reservationId, ]); // [3] Use upstream reservation if provided, else reserve a fresh IP. $picked = $this->reservationId ? $this->loadReservation($this->reservationId, $this->reservedIpPoolId, $logger) : $this->reserveIp($networkId, $hostId, $dns); if (!$picked) { $logger->error('no free IPs in pool', [ 'network_id' => $networkId, 'attempt' => $attempt, ]); break; } $reservation = $picked['reservation']; $ipPoolId = $picked['ip_pool_id']; $ip = $reservation->ip; $gateway = (string) $picked['gateway']; $cidr = (string) $picked['cidr']; // [4] Assign IP into VM guest OS via PS1 script (Linux uses assignIPLinux.ps1) $assignOutput = $ssh->assignIp( vmId: $this->vmId, ip: $ip, gateway: $gateway, cidr: $cidr, vcenterHost: $vcenterHost, vcenterUser: $vcenterUser, vcenterPassword: $vcenterPass, guestPassword: $this->guestPassword, osFamily: $this->osFamily, dns: $dns, guestUser: $guestUser, ); // [5] If assign failed → release and try next IP if (!$this->isSuccess($assignOutput)) { $logger->warning('assignIp FAIL', [ 'ip' => $ip, 'attempt' => $attempt, 'output' => trim($assignOutput), ]); $telegram->SendMessage( "⚠️ AssignIP thất bại\n" . "🖥️ VM: {$this->vmId} | IP: {$ip}\n" . "🔁 Lần thử: {$attempt}/{$maxAttempts}\n" . "📋 Output: " . htmlspecialchars(trim($assignOutput) ?: '(trống)') . "\n" . "🕒 " . Carbon::now()->format('d/m/Y H:i:s') ); $this->releaseIp($reservation, $ipPoolId, $logger); continue; } if (strtoupper($this->osFamily) === VmProvisionService::OS_WINDOWS) { $this->persistIp($reservation, $moduleVmware, $ipPoolId, $serviceId); $logger->success('Windows IP assigned', [ 'module_vmware_id' => $this->moduleVmwareId, 'vm_id' => $this->vmId, 'ip' => $ip, 'attempt' => $attempt, ]); ChangePasswordViaVsphereJob::dispatch( $this->moduleVmwareId, $this->vmId, $this->osFamily, ); return; } // [6] Verify IP is active inside guest OS via PS1 script $verifyOutput = $ssh->verifyIpInGuest( vmId: $this->vmId, ip: $ip, vcenterHost: $vcenterHost, vcenterUser: $vcenterUser, vcenterPassword: $vcenterPass, guestPassword: $this->guestPassword, osFamily: $this->osFamily, guestUser: $guestUser, ); // [7] If verification failed → release and try next IP if (!$this->isSuccess($verifyOutput)) { $logger->warning('verifyIpInGuest FAIL', [ 'ip' => $ip, 'attempt' => $attempt, 'output' => trim($verifyOutput), ]); $this->releaseIp($reservation, $ipPoolId, $logger); continue; } // [8] SUCCESS — persist confirmed IP to module_vmware + virtual_machines + ip_pools $this->persistIp($reservation, $moduleVmware, $ipPoolId, $serviceId); $logger->success('IP assigned and verified', [ 'module_vmware_id' => $this->moduleVmwareId, 'vm_id' => $this->vmId, 'ip' => $ip, 'attempt' => $attempt, ]); // CPU/RAM config was already applied by UpdateVmConfigJob before IP assignment. ChangePasswordViaVsphereJob::dispatch( $this->moduleVmwareId, $this->vmId, $this->osFamily, ); return; } // [9] All attempts exhausted $logger->error('all attempts failed — marking VM inactive', [ 'module_vmware_id' => $this->moduleVmwareId, 'vm_id' => $this->vmId, 'attempts' => $maxAttempts, ]); ModuleVmware::where('id', $this->moduleVmwareId)->update(['status' => 0]); Service::where('vm_id', $this->moduleVmwareId)->update(['status' => 'failed']); // Notify admin via Telegram with order/customer context try { $vm = ModuleVmware::with(['order.customer', 'order.orderItems.product'])->find($this->moduleVmwareId); $customer = $vm?->order?->customer; $order = $vm?->order; $product = $order?->orderItems?->first()?->getProductAttribute(); $telegram->SendMessage( "🚨 Khởi tạo VM thất bại hoàn toàn\n" . "━━━━━━━━━━━━━━━━━━━━\n" . "🖥️ VM ID: {$this->vmId}\n" . "📦 Sản phẩm: " . ($product?->name ?? 'N/A') . "\n" . "🧾 Đơn hàng: #" . ($order?->order_num ?? 'N/A') . "\n" . "👤 Khách hàng: " . ($customer ? trim($customer->firstname . ' ' . $customer->lastname) : 'N/A') . "\n" . "📧 Email: " . ($customer?->email ?? 'N/A') . "\n" . "🔁 Số lần thử: {$maxAttempts}\n" . "⚡ Trạng thái: VM đã bị đánh dấu inactive\n" . "🕒 " . Carbon::now()->format('d/m/Y H:i:s') . "\n" . "━━━━━━━━━━━━━━━━━━━━\n" . "⚠️ Vui lòng kiểm tra và xử lý thủ công!" ); } catch (\Throwable $e) { $logger->warning('could not send Telegram failure notification', ['error' => $e->getMessage()]); } // Attempt to recover IP from vSphere guest identity for admin visibility only. // Do not send credentials here: password has not been rotated yet. try { $moduleVmware = ModuleVmware::find($this->moduleVmwareId); if ($moduleVmware && !$moduleVmware->ip) { $vcenter = $moduleVmware->vCenter ?? vCenter::first(); if ($vcenter && $vCenterService->loginServer($vcenter)) { $guestInfo = $vCenterService->GetIPAddressVirtualMachine($this->vmId); if (isset($guestInfo['error']) && $guestInfo['error'] === false) { $guestIp = $guestInfo['data']['ip_address'] ?? null; if ($guestIp) { $moduleVmware->update(['ip' => $guestIp, 'status' => 1]); $logger->step('recovered IP from guest identity', [ 'module_vmware_id' => $this->moduleVmwareId, 'ip' => $guestIp, ]); } } } } } catch (\Throwable $e) { $logger->error('failed to recover IP after exhausted attempts', [ 'module_vmware_id' => $this->moduleVmwareId, 'error' => $e->getMessage(), ]); } } public function failed(\Throwable $e): void { Log::error('AssignIpToGuestJob: job failed with exception', [ 'module_vmware_id' => $this->moduleVmwareId, 'vm_id' => $this->vmId, 'error' => $e->getMessage(), ]); ModuleVmware::where('id', $this->moduleVmwareId)->update(['status' => 0]); Service::where('vm_id', $this->moduleVmwareId)->update(['status' => 'failed']); } // ───────────────────────────────────────────────────────────────────────── // IP reservation helpers // ───────────────────────────────────────────────────────────────────────── /** * Reserve a free IP from the pool inside a DB transaction with row-level locks. * * Strategy: * 1. Lock all existing ModuleIpAddress rows for the network → prevents double-pick * 2. Lock the chosen IpPool row → prevents concurrent reservations * 3. Create a pending ModuleIpAddress (status=0) to hold the slot * * Returns the pending ModuleIpAddress record, or null if pool is exhausted. */ /** * @return array{reservation:ModuleIpAddress,ip_pool_id:int,gateway:string,cidr:int}|null */ private function reserveIp( int $networkId, int $hostId, string $dns, ): ?array { return DB::transaction(function () use ($networkId, $hostId, $dns) { // ip_pools.status is the single source of truth for availability. $ipPool = IpPool::where('module_network_id', $networkId) ->whereIn('status', ['free', 'reserved']) ->lockForUpdate() ->inRandomOrder() ->first(); if (!$ipPool) { return null; } // Hold the pool slot $ipPool->update(['status' => 'reserved']); $cidr = (int) ($ipPool->cidr ?: 24); $gateway = (string) ($ipPool->gateway ?? ''); $netmask = $ipPool->netmask ?: $this->cidrToNetmask($cidr); // module_ip_address mirrors ip_pools — update the existing synced row // (created when the IP was added to ip_pools) instead of inserting a new one. $reservation = ModuleIpAddress::where('ip', $ipPool->ip_address) ->where('module_network_id', $networkId) ->lockForUpdate() ->first(); $attrs = [ 'vm_name' => '__pending__', 'module_host_id' => $hostId, 'module_network_id' => $networkId, 'ip' => $ipPool->ip_address, 'gateway' => $gateway, 'netmask' => $netmask, 'dns' => $dns, 'status' => 0, // 0 = pending/reserved, not yet confirmed ]; if ($reservation) { $reservation->update($attrs); } else { $reservation = ModuleIpAddress::create($attrs); } return [ 'reservation' => $reservation, 'ip_pool_id' => $ipPool->id, 'gateway' => $gateway, 'cidr' => $cidr, ]; }); } /** * Load a reservation that was created upstream (e.g. by CreateVirtualMachineJob). * Returns the same shape as reserveIp() so the rest of the flow is identical. * * @return array{reservation:ModuleIpAddress,ip_pool_id:int,gateway:string,cidr:int}|null */ private function loadReservation(int $reservationId, ?int $ipPoolId, VmProvisionLogger $logger): ?array { $reservation = ModuleIpAddress::find($reservationId); if (!$reservation) { $logger->error('upstream reservation missing', [ 'reservation_id' => $reservationId, ]); return null; } $ipPool = $ipPoolId ? IpPool::find($ipPoolId) : null; $cidr = (int) ($ipPool->cidr ?? 24); $gateway = (string) ($reservation->gateway ?: ($ipPool->gateway ?? '')); return [ 'reservation' => $reservation, 'ip_pool_id' => $ipPoolId ?? 0, 'gateway' => $gateway, 'cidr' => $cidr ?: 24, ]; } /** * Release a failed reservation by resetting the mirrored ModuleIpAddress row. * This returns the IP to the available pool. */ private function releaseIp(ModuleIpAddress $reservation, ?int $ipPoolId, VmProvisionLogger $logger): void { try { $reservation->update(['vm_name' => null, 'status' => 0]); if ($ipPoolId) { IpPool::where('id', $ipPoolId)->update(['status' => 'free']); } $logger->step('IP reservation released', ['ip' => $reservation->ip]); } catch (\Throwable $e) { $logger->warning('could not release IP reservation', [ 'ip' => $reservation->ip, 'error' => $e->getMessage(), ]); } } /** * Confirm a successful IP assignment: * - Update ModuleIpAddress: status → 1 (assigned), vm_name → real name * - Update ModuleVmware: ip, dns, module_ip_address FK * - Update VirtualMachines: ip, ip_pool_id FK (matched by vmId) * * IP is only written to DB AFTER guest-side verification passes. */ private function persistIp( ModuleIpAddress $reservation, ModuleVmware $moduleVmware, ?int $ipPoolId, ?int $serviceId, ): void { DB::transaction(function () use ($reservation, $moduleVmware, $ipPoolId, $serviceId) { $reservation->update([ 'vm_name' => $moduleVmware->name ?? ('VM-' . $this->vmId), 'status' => 1, // assigned ]); $moduleVmware->update([ 'ip' => $reservation->ip, 'dns' => $reservation->dns, 'module_ip_address' => $reservation->id, ]); // Sync to virtual_machines — matched by vSphere vmId VirtualMachines::where('vm_id', $this->vmId)->update([ 'ip' => $reservation->ip, 'ip_pool_id' => $ipPoolId, ]); // Flip ip_pools row to assigned and link service if ($ipPoolId) { IpPool::where('id', $ipPoolId)->update([ 'status' => 'assigned', 'service_id' => $serviceId, ]); } // Touch services row so server_name reflects the provisioned VM. // Note: services.vm_id is an FK to module_vmware.id (already set by VmServiceSyncService) — do not overwrite with vSphere vmId string. if ($serviceId) { Service::where('id', $serviceId)->update([ 'server_name' => $moduleVmware->name, ]); } }); } /** Convert CIDR prefix length to dotted-decimal netmask (e.g. 24 → 255.255.255.0). */ private function cidrToNetmask(int $cidr): string { $cidr = max(0, min(32, $cidr)); return long2ip(-1 << (32 - $cidr)); } // ───────────────────────────────────────────────────────────────────────── // Micro-helpers // ───────────────────────────────────────────────────────────────────────── /** PS1 scripts return JSON: {"status":"ok"} or {"status":"error","message":"..."} */ private function isSuccess(string $output): bool { if (str_contains(strtoupper(trim($output)), 'SUCCESS')) { return true; } $data = json_decode(trim($output), true); return is_array($data) && ($data['status'] ?? '') === 'ok'; } }