503 lines
21 KiB
PHP
503 lines
21 KiB
PHP
<?php
|
||
|
||
declare(strict_types=1);
|
||
|
||
namespace App\Jobs\VirtualMachine\Provision;
|
||
|
||
use App\Http\Facade\SignRSA;
|
||
use App\Http\Services\Telegram;
|
||
use App\Models\IpPool;
|
||
use App\Models\ModuleIpAddress;
|
||
use App\Models\ModuleVmware;
|
||
use App\Models\Os;
|
||
use App\Models\Service;
|
||
use App\Models\VirtualMachines;
|
||
use App\Http\Services\vCenter as vCenterService;
|
||
use App\Models\vCenter;
|
||
use App\Services\VirtualMachine\VmProvisionLogger;
|
||
use App\Services\VmProvisionService;
|
||
use App\Services\VsphereHostSshService;
|
||
use Carbon\Carbon;
|
||
use Illuminate\Bus\Queueable;
|
||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||
use Illuminate\Foundation\Bus\Dispatchable;
|
||
use Illuminate\Queue\InteractsWithQueue;
|
||
use Illuminate\Queue\SerializesModels;
|
||
use Illuminate\Support\Facades\DB;
|
||
use Illuminate\Support\Facades\Log;
|
||
|
||
/**
|
||
* AssignIpToGuestJob
|
||
*
|
||
* Assigns a real static IP to the VM guest OS and verifies it is active.
|
||
* Only persists the IP to DB (module_vmware + virtual_machines) after
|
||
* successful guest-side verification, then dispatches ChangePasswordViaVsphereJob.
|
||
*
|
||
* Flow:
|
||
* [1] Resolve network config from order item → product → ipAddress template
|
||
* [2] for attempt 1..MAX_ATTEMPTS:
|
||
* [3] Pick a free IP with DB transaction + lock (reserveIp)
|
||
* [4] Call assignIPLinux.ps1 / assignIPWindows.ps1 via SSH
|
||
* [5] On FAIL → release IP, try next
|
||
* [6] Call verifyIPLinux.ps1 / verifyIPWindows.ps1 via SSH
|
||
* [7] On FAIL → release IP, try next
|
||
* [8] SUCCESS → persistIp() → dispatch ChangePasswordViaVsphereJob
|
||
* [9] All attempts exhausted → mark VM inactive (status=0)
|
||
*/
|
||
class AssignIpToGuestJob implements ShouldQueue
|
||
{
|
||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||
|
||
/** All retries handled internally — no Laravel-level retry needed */
|
||
public int $tries = 1;
|
||
|
||
/** 5 attempts × ~90 s each + overhead */
|
||
public int $timeout = 600;
|
||
|
||
private const MAX_ATTEMPTS = 5;
|
||
|
||
public function __construct(
|
||
public readonly int $moduleVmwareId,
|
||
public readonly string $vmId,
|
||
public readonly string $osFamily, // VmProvisionService::OS_LINUX | OS_WINDOWS
|
||
public readonly string $guestPassword = VmProvisionService::TEMPLATE_PASSWORD,
|
||
public readonly ?int $reservationId = null,
|
||
public readonly ?int $reservedIpPoolId = null,
|
||
) {}
|
||
|
||
// ─────────────────────────────────────────────────────────────────────────
|
||
|
||
public function handle(VsphereHostSshService $ssh, vCenterService $vCenterService, VmProvisionLogger $logger, Telegram $telegram): void
|
||
{
|
||
$logger->for('AssignIpToGuestJob');
|
||
|
||
$logger->step('start', [
|
||
'module_vmware_id' => $this->moduleVmwareId,
|
||
'vm_id' => $this->vmId,
|
||
'os_family' => $this->osFamily,
|
||
]);
|
||
|
||
// Load ModuleVmware with the relations needed to derive network config
|
||
$moduleVmware = ModuleVmware::with([
|
||
'order.orderItems.product.ipAddress',
|
||
'vcenter',
|
||
])->find($this->moduleVmwareId);
|
||
|
||
if (!$moduleVmware) {
|
||
$logger->error('ModuleVmware not found', [
|
||
'module_vmware_id' => $this->moduleVmwareId,
|
||
]);
|
||
return;
|
||
}
|
||
|
||
// DNS + network selector still come from product template; gateway + CIDR
|
||
// are now read from the chosen ip_pools row at reservation time.
|
||
$orderItem = $moduleVmware->order?->orderItems?->first();
|
||
$osModel = $orderItem?->os_id ? Os::with('osGroup.host')->find($orderItem->os_id) : null;
|
||
|
||
$networkId = $orderItem?->product?->ipAddress?->module_network_id ?? 1;
|
||
$dns = $orderItem?->product?->ipAddress?->dns ?? '8.8.8.8';
|
||
$hostId = $osModel?->osGroup?->host?->id ?? 1;
|
||
$serviceId = Service::where('vm_id', $this->moduleVmwareId)->value('id');
|
||
|
||
// vCenter credentials
|
||
$vcenter = $moduleVmware->vcenter ?? vCenter::first();
|
||
$vcenterHost = $vcenter->host;
|
||
$vcenterUser = $vcenter->user;
|
||
$vcenterPass = SignRSA::DecryptFunc($vcenter->password);
|
||
|
||
// Guest OS default admin user per OS family
|
||
$guestUser = strtoupper($this->osFamily) === 'WINDOWS' ? 'Administrator' : 'root';
|
||
|
||
// ─── Attempt loop ────────────────────────────────────────────────────
|
||
// If reservationId is supplied, the IP was reserved upstream (CreateVirtualMachineJob)
|
||
// and baked into the VM name — we can't swap IPs on retry, so we run a single attempt.
|
||
$maxAttempts = $this->reservationId ? 1 : self::MAX_ATTEMPTS;
|
||
|
||
for ($attempt = 1; $attempt <= $maxAttempts; $attempt++) {
|
||
$logger->step('attempt', [
|
||
'attempt' => $attempt,
|
||
'module_vmware_id' => $this->moduleVmwareId,
|
||
'network_id' => $networkId,
|
||
'preserved' => (bool) $this->reservationId,
|
||
]);
|
||
|
||
// [3] Use upstream reservation if provided, else reserve a fresh IP.
|
||
$picked = $this->reservationId
|
||
? $this->loadReservation($this->reservationId, $this->reservedIpPoolId, $logger)
|
||
: $this->reserveIp($networkId, $hostId, $dns);
|
||
|
||
if (!$picked) {
|
||
$logger->error('no free IPs in pool', [
|
||
'network_id' => $networkId,
|
||
'attempt' => $attempt,
|
||
]);
|
||
break;
|
||
}
|
||
|
||
$reservation = $picked['reservation'];
|
||
$ipPoolId = $picked['ip_pool_id'];
|
||
$ip = $reservation->ip;
|
||
$gateway = (string) $picked['gateway'];
|
||
$cidr = (string) $picked['cidr'];
|
||
|
||
// [4] Assign IP into VM guest OS via PS1 script (Linux uses assignIPLinux.ps1)
|
||
$assignOutput = $ssh->assignIp(
|
||
vmId: $this->vmId,
|
||
ip: $ip,
|
||
gateway: $gateway,
|
||
cidr: $cidr,
|
||
vcenterHost: $vcenterHost,
|
||
vcenterUser: $vcenterUser,
|
||
vcenterPassword: $vcenterPass,
|
||
guestPassword: $this->guestPassword,
|
||
osFamily: $this->osFamily,
|
||
dns: $dns,
|
||
guestUser: $guestUser,
|
||
);
|
||
|
||
// [5] If assign failed → release and try next IP
|
||
if (!$this->isSuccess($assignOutput)) {
|
||
$logger->warning('assignIp FAIL', [
|
||
'ip' => $ip,
|
||
'attempt' => $attempt,
|
||
'output' => trim($assignOutput),
|
||
]);
|
||
$telegram->SendMessage(
|
||
"⚠️ <b>AssignIP thất bại</b>\n" .
|
||
"🖥️ VM: <code>{$this->vmId}</code> | IP: <code>{$ip}</code>\n" .
|
||
"🔁 Lần thử: {$attempt}/{$maxAttempts}\n" .
|
||
"📋 Output: <code>" . htmlspecialchars(trim($assignOutput) ?: '(trống)') . "</code>\n" .
|
||
"🕒 " . Carbon::now()->format('d/m/Y H:i:s')
|
||
);
|
||
$this->releaseIp($reservation, $ipPoolId, $logger);
|
||
continue;
|
||
}
|
||
|
||
if (strtoupper($this->osFamily) === VmProvisionService::OS_WINDOWS) {
|
||
$this->persistIp($reservation, $moduleVmware, $ipPoolId, $serviceId);
|
||
|
||
$logger->success('Windows IP assigned', [
|
||
'module_vmware_id' => $this->moduleVmwareId,
|
||
'vm_id' => $this->vmId,
|
||
'ip' => $ip,
|
||
'attempt' => $attempt,
|
||
]);
|
||
|
||
ChangePasswordViaVsphereJob::dispatch(
|
||
$this->moduleVmwareId,
|
||
$this->vmId,
|
||
$this->osFamily,
|
||
);
|
||
|
||
return;
|
||
}
|
||
|
||
// [6] Verify IP is active inside guest OS via PS1 script
|
||
$verifyOutput = $ssh->verifyIpInGuest(
|
||
vmId: $this->vmId,
|
||
ip: $ip,
|
||
vcenterHost: $vcenterHost,
|
||
vcenterUser: $vcenterUser,
|
||
vcenterPassword: $vcenterPass,
|
||
guestPassword: $this->guestPassword,
|
||
osFamily: $this->osFamily,
|
||
guestUser: $guestUser,
|
||
);
|
||
|
||
// [7] If verification failed → release and try next IP
|
||
if (!$this->isSuccess($verifyOutput)) {
|
||
$logger->warning('verifyIpInGuest FAIL', [
|
||
'ip' => $ip,
|
||
'attempt' => $attempt,
|
||
'output' => trim($verifyOutput),
|
||
]);
|
||
$this->releaseIp($reservation, $ipPoolId, $logger);
|
||
continue;
|
||
}
|
||
|
||
// [8] SUCCESS — persist confirmed IP to module_vmware + virtual_machines + ip_pools
|
||
$this->persistIp($reservation, $moduleVmware, $ipPoolId, $serviceId);
|
||
|
||
$logger->success('IP assigned and verified', [
|
||
'module_vmware_id' => $this->moduleVmwareId,
|
||
'vm_id' => $this->vmId,
|
||
'ip' => $ip,
|
||
'attempt' => $attempt,
|
||
]);
|
||
|
||
// Dispatch password change now that IP is confirmed
|
||
ChangePasswordViaVsphereJob::dispatch(
|
||
$this->moduleVmwareId,
|
||
$this->vmId,
|
||
$this->osFamily,
|
||
);
|
||
|
||
return;
|
||
}
|
||
|
||
// [9] All attempts exhausted
|
||
$logger->error('all attempts failed — marking VM inactive', [
|
||
'module_vmware_id' => $this->moduleVmwareId,
|
||
'vm_id' => $this->vmId,
|
||
'attempts' => $maxAttempts,
|
||
]);
|
||
|
||
ModuleVmware::where('id', $this->moduleVmwareId)->update(['status' => 0]);
|
||
Service::where('vm_id', $this->moduleVmwareId)->update(['status' => 'failed']);
|
||
|
||
// Notify admin via Telegram with order/customer context
|
||
try {
|
||
$vm = ModuleVmware::with(['order.customer', 'order.orderItems.product'])->find($this->moduleVmwareId);
|
||
$customer = $vm?->order?->customer;
|
||
$order = $vm?->order;
|
||
$product = $order?->orderItems?->first()?->getProductAttribute();
|
||
|
||
$telegram->SendMessage(
|
||
"🚨 <b>Khởi tạo VM thất bại hoàn toàn</b>\n" .
|
||
"━━━━━━━━━━━━━━━━━━━━\n" .
|
||
"🖥️ VM ID: <code>{$this->vmId}</code>\n" .
|
||
"📦 Sản phẩm: " . ($product?->name ?? 'N/A') . "\n" .
|
||
"🧾 Đơn hàng: #" . ($order?->order_num ?? 'N/A') . "\n" .
|
||
"👤 Khách hàng: " . ($customer ? trim($customer->firstname . ' ' . $customer->lastname) : 'N/A') . "\n" .
|
||
"📧 Email: " . ($customer?->email ?? 'N/A') . "\n" .
|
||
"🔁 Số lần thử: {$maxAttempts}\n" .
|
||
"⚡ Trạng thái: VM đã bị đánh dấu inactive\n" .
|
||
"🕒 " . Carbon::now()->format('d/m/Y H:i:s') . "\n" .
|
||
"━━━━━━━━━━━━━━━━━━━━\n" .
|
||
"⚠️ Vui lòng kiểm tra và xử lý thủ công!"
|
||
);
|
||
} catch (\Throwable $e) {
|
||
$logger->warning('could not send Telegram failure notification', ['error' => $e->getMessage()]);
|
||
}
|
||
|
||
// Attempt to recover IP from vSphere guest identity for admin visibility only.
|
||
// Do not send credentials here: password has not been rotated yet.
|
||
try {
|
||
$moduleVmware = ModuleVmware::find($this->moduleVmwareId);
|
||
if ($moduleVmware && !$moduleVmware->ip) {
|
||
$vcenter = $moduleVmware->vCenter ?? vCenter::first();
|
||
if ($vcenter && $vCenterService->loginServer($vcenter)) {
|
||
$guestInfo = $vCenterService->GetIPAddressVirtualMachine($this->vmId);
|
||
if (isset($guestInfo['error']) && $guestInfo['error'] === false) {
|
||
$guestIp = $guestInfo['data']['ip_address'] ?? null;
|
||
if ($guestIp) {
|
||
$moduleVmware->update(['ip' => $guestIp, 'status' => 1]);
|
||
$logger->step('recovered IP from guest identity', [
|
||
'module_vmware_id' => $this->moduleVmwareId,
|
||
'ip' => $guestIp,
|
||
]);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} catch (\Throwable $e) {
|
||
$logger->error('failed to recover IP after exhausted attempts', [
|
||
'module_vmware_id' => $this->moduleVmwareId,
|
||
'error' => $e->getMessage(),
|
||
]);
|
||
}
|
||
}
|
||
|
||
public function failed(\Throwable $e): void
|
||
{
|
||
Log::error('AssignIpToGuestJob: job failed with exception', [
|
||
'module_vmware_id' => $this->moduleVmwareId,
|
||
'vm_id' => $this->vmId,
|
||
'error' => $e->getMessage(),
|
||
]);
|
||
|
||
ModuleVmware::where('id', $this->moduleVmwareId)->update(['status' => 0]);
|
||
Service::where('vm_id', $this->moduleVmwareId)->update(['status' => 'failed']);
|
||
}
|
||
|
||
// ─────────────────────────────────────────────────────────────────────────
|
||
// IP reservation helpers
|
||
// ─────────────────────────────────────────────────────────────────────────
|
||
|
||
/**
|
||
* Reserve a free IP from the pool inside a DB transaction with row-level locks.
|
||
*
|
||
* Strategy:
|
||
* 1. Lock all existing ModuleIpAddress rows for the network → prevents double-pick
|
||
* 2. Lock the chosen IpPool row → prevents concurrent reservations
|
||
* 3. Create a pending ModuleIpAddress (status=0) to hold the slot
|
||
*
|
||
* Returns the pending ModuleIpAddress record, or null if pool is exhausted.
|
||
*/
|
||
/**
|
||
* @return array{reservation:ModuleIpAddress,ip_pool_id:int,gateway:string,cidr:int}|null
|
||
*/
|
||
private function reserveIp(
|
||
int $networkId,
|
||
int $hostId,
|
||
string $dns,
|
||
): ?array {
|
||
return DB::transaction(function () use ($networkId, $hostId, $dns) {
|
||
// Lock existing rows to prevent another job from picking the same IP
|
||
$usedIps = ModuleIpAddress::where('module_network_id', $networkId)
|
||
->lockForUpdate()
|
||
->pluck('ip')
|
||
->toArray();
|
||
|
||
$ipPool = IpPool::where('module_network_id', $networkId)
|
||
->whereIn('status', ['free', 'reserved'])
|
||
->whereNotIn('ip_address', $usedIps)
|
||
->lockForUpdate()
|
||
->inRandomOrder()
|
||
->first();
|
||
|
||
if (!$ipPool) {
|
||
return null;
|
||
}
|
||
|
||
// Hold the pool slot
|
||
$ipPool->update(['status' => 'reserved']);
|
||
|
||
$cidr = (int) ($ipPool->cidr ?: 24);
|
||
$gateway = (string) ($ipPool->gateway ?? '');
|
||
$netmask = $this->cidrToNetmask($cidr);
|
||
|
||
// Create a pending reservation — acts as a distributed lock on this IP
|
||
$reservation = ModuleIpAddress::create([
|
||
'vm_name' => '__pending__',
|
||
'module_host_id' => $hostId,
|
||
'module_network_id' => $networkId,
|
||
'ip' => $ipPool->ip_address,
|
||
'gateway' => $gateway,
|
||
'netmask' => $netmask,
|
||
'dns' => $dns,
|
||
'status' => 0, // 0 = pending/reserved, not yet confirmed
|
||
]);
|
||
|
||
return [
|
||
'reservation' => $reservation,
|
||
'ip_pool_id' => $ipPool->id,
|
||
'gateway' => $gateway,
|
||
'cidr' => $cidr,
|
||
];
|
||
});
|
||
}
|
||
|
||
/**
|
||
* Load a reservation that was created upstream (e.g. by CreateVirtualMachineJob).
|
||
* Returns the same shape as reserveIp() so the rest of the flow is identical.
|
||
*
|
||
* @return array{reservation:ModuleIpAddress,ip_pool_id:int,gateway:string,cidr:int}|null
|
||
*/
|
||
private function loadReservation(int $reservationId, ?int $ipPoolId, VmProvisionLogger $logger): ?array
|
||
{
|
||
$reservation = ModuleIpAddress::find($reservationId);
|
||
if (!$reservation) {
|
||
$logger->error('upstream reservation missing', [
|
||
'reservation_id' => $reservationId,
|
||
]);
|
||
return null;
|
||
}
|
||
|
||
$ipPool = $ipPoolId ? IpPool::find($ipPoolId) : null;
|
||
$cidr = (int) ($ipPool->cidr ?? 24);
|
||
$gateway = (string) ($reservation->gateway ?: ($ipPool->gateway ?? ''));
|
||
|
||
return [
|
||
'reservation' => $reservation,
|
||
'ip_pool_id' => $ipPoolId ?? 0,
|
||
'gateway' => $gateway,
|
||
'cidr' => $cidr ?: 24,
|
||
];
|
||
}
|
||
|
||
/**
|
||
* Release a failed reservation by deleting the pending ModuleIpAddress row.
|
||
* This returns the IP to the available pool.
|
||
*/
|
||
private function releaseIp(ModuleIpAddress $reservation, ?int $ipPoolId, VmProvisionLogger $logger): void
|
||
{
|
||
try {
|
||
$reservation->delete();
|
||
if ($ipPoolId) {
|
||
IpPool::where('id', $ipPoolId)->update(['status' => 'free']);
|
||
}
|
||
$logger->step('IP reservation released', ['ip' => $reservation->ip]);
|
||
} catch (\Throwable $e) {
|
||
$logger->warning('could not release IP reservation', [
|
||
'ip' => $reservation->ip,
|
||
'error' => $e->getMessage(),
|
||
]);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Confirm a successful IP assignment:
|
||
* - Update ModuleIpAddress: status → 1 (assigned), vm_name → real name
|
||
* - Update ModuleVmware: ip, dns, module_ip_address FK
|
||
* - Update VirtualMachines: ip, ip_pool_id FK (matched by vmId)
|
||
*
|
||
* IP is only written to DB AFTER guest-side verification passes.
|
||
*/
|
||
private function persistIp(
|
||
ModuleIpAddress $reservation,
|
||
ModuleVmware $moduleVmware,
|
||
?int $ipPoolId,
|
||
?int $serviceId,
|
||
): void {
|
||
DB::transaction(function () use ($reservation, $moduleVmware, $ipPoolId, $serviceId) {
|
||
$reservation->update([
|
||
'vm_name' => $moduleVmware->name ?? ('VM-' . $this->vmId),
|
||
'status' => 1, // assigned
|
||
]);
|
||
|
||
$moduleVmware->update([
|
||
'ip' => $reservation->ip,
|
||
'dns' => $reservation->dns,
|
||
'module_ip_address' => $reservation->id,
|
||
]);
|
||
|
||
// Sync to virtual_machines — matched by vSphere vmId
|
||
VirtualMachines::where('vm_id', $this->vmId)->update([
|
||
'ip' => $reservation->ip,
|
||
'ip_pool_id' => $ipPoolId,
|
||
]);
|
||
|
||
// Flip ip_pools row to assigned and link service
|
||
if ($ipPoolId) {
|
||
IpPool::where('id', $ipPoolId)->update([
|
||
'status' => 'assigned',
|
||
'service_id' => $serviceId,
|
||
]);
|
||
}
|
||
|
||
// Touch services row so server_name reflects the provisioned VM.
|
||
// Note: services.vm_id is an FK to module_vmware.id (already set by VmServiceSyncService) — do not overwrite with vSphere vmId string.
|
||
if ($serviceId) {
|
||
Service::where('id', $serviceId)->update([
|
||
'server_name' => $moduleVmware->name,
|
||
]);
|
||
}
|
||
});
|
||
}
|
||
|
||
/** Convert CIDR prefix length to dotted-decimal netmask (e.g. 24 → 255.255.255.0). */
|
||
private function cidrToNetmask(int $cidr): string
|
||
{
|
||
$cidr = max(0, min(32, $cidr));
|
||
return long2ip(-1 << (32 - $cidr));
|
||
}
|
||
|
||
// ─────────────────────────────────────────────────────────────────────────
|
||
// Micro-helpers
|
||
// ─────────────────────────────────────────────────────────────────────────
|
||
|
||
/** PS1 scripts return JSON: {"status":"ok"} or {"status":"error","message":"..."} */
|
||
private function isSuccess(string $output): bool
|
||
{
|
||
if (str_contains(strtoupper(trim($output)), 'SUCCESS')) {
|
||
return true;
|
||
}
|
||
|
||
$data = json_decode(trim($output), true);
|
||
return is_array($data) && ($data['status'] ?? '') === 'ok';
|
||
}
|
||
|
||
}
|