Refactor VM provisioning pipeline into focused services and add IP pool
management
This commit is contained in:
baodev0
@@ -9,8 +9,11 @@ use App\Models\IpPool;
|
||||
use App\Models\ModuleIpAddress;
|
||||
use App\Models\ModuleVmware;
|
||||
use App\Models\Os;
|
||||
use App\Models\Service;
|
||||
use App\Models\VirtualMachines;
|
||||
use App\Http\Services\vCenter as vCenterService;
|
||||
use App\Models\vCenter;
|
||||
use App\Services\VmProvisionService;
|
||||
use App\Services\VsphereHostSshService;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
@@ -24,7 +27,8 @@ use Illuminate\Support\Facades\Log;
|
||||
* AssignIpToGuestJob
|
||||
*
|
||||
* Assigns a real static IP to the VM guest OS and verifies it is active.
|
||||
* Only persists the IP to DB after successful guest-side verification.
|
||||
* Only persists the IP to DB (module_vmware + virtual_machines) after
|
||||
* successful guest-side verification, then dispatches ChangePasswordViaVsphereJob.
|
||||
*
|
||||
* Flow:
|
||||
* [1] Resolve network config from order item → product → ipAddress template
|
||||
@@ -34,7 +38,7 @@ use Illuminate\Support\Facades\Log;
|
||||
* [5] On FAIL → release IP, try next
|
||||
* [6] Call verifyIPLinux.ps1 / verifyIPWindows.ps1 via SSH
|
||||
* [7] On FAIL → release IP, try next
|
||||
* [8] SUCCESS → persistIp() → dispatch SendVmCredentialsEmailJob
|
||||
* [8] SUCCESS → persistIp() → dispatch ChangePasswordViaVsphereJob
|
||||
* [9] All attempts exhausted → mark VM inactive (status=0)
|
||||
*/
|
||||
class AssignIpToGuestJob implements ShouldQueue
|
||||
@@ -52,8 +56,8 @@ class AssignIpToGuestJob implements ShouldQueue
|
||||
public function __construct(
|
||||
public readonly int $moduleVmwareId,
|
||||
public readonly string $vmId,
|
||||
public readonly string $osFamily, // VmProvisionService::OS_LINUX | OS_WINDOWS
|
||||
public readonly string $guestPassword, // New password from ChangePasswordViaVsphereJob
|
||||
public readonly string $osFamily, // VmProvisionService::OS_LINUX | OS_WINDOWS
|
||||
public readonly string $guestPassword = VmProvisionService::TEMPLATE_PASSWORD,
|
||||
) {}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
@@ -79,16 +83,15 @@ class AssignIpToGuestJob implements ShouldQueue
|
||||
return;
|
||||
}
|
||||
|
||||
// Resolve network config: product template IP → gateway, netmask, dns, networkId
|
||||
// DNS + network selector still come from product template; gateway + CIDR
|
||||
// are now read from the chosen ip_pools row at reservation time.
|
||||
$orderItem = $moduleVmware->order?->orderItems?->first();
|
||||
$osModel = $orderItem?->os_id ? Os::with('osGroup.host')->find($orderItem->os_id) : null;
|
||||
|
||||
$networkId = $orderItem?->product?->ipAddress?->module_network_id ?? 1;
|
||||
$gateway = $orderItem?->product?->ipAddress?->gateway ?? '0.0.0.0';
|
||||
$netmask = $orderItem?->product?->ipAddress?->netmask ?? '255.255.255.0';
|
||||
$dns = $orderItem?->product?->ipAddress?->dns ?? '8.8.8.8';
|
||||
$hostId = $osModel?->osGroup?->host?->id ?? 1;
|
||||
$cidr = (string) $this->netmaskToCidr($netmask);
|
||||
$dns = $orderItem?->product?->ipAddress?->dns ?? '8.8.8.8';
|
||||
$hostId = $osModel?->osGroup?->host?->id ?? 1;
|
||||
$serviceId = Service::where('vm_id', $this->moduleVmwareId)->value('id');
|
||||
|
||||
// vCenter credentials
|
||||
$vcenter = $moduleVmware->vcenter ?? vCenter::first();
|
||||
@@ -109,9 +112,9 @@ class AssignIpToGuestJob implements ShouldQueue
|
||||
]);
|
||||
|
||||
// [3] Reserve a free IP with transaction + row lock
|
||||
$reservation = $this->reserveIp($networkId, $hostId, $gateway, $netmask, $dns);
|
||||
$picked = $this->reserveIp($networkId, $hostId, $dns);
|
||||
|
||||
if (!$reservation) {
|
||||
if (!$picked) {
|
||||
Log::error('AssignIpToGuestJob: no free IPs in pool', [
|
||||
'network_id' => $networkId,
|
||||
'attempt' => $attempt,
|
||||
@@ -119,9 +122,13 @@ class AssignIpToGuestJob implements ShouldQueue
|
||||
break; // No point retrying — no IPs available
|
||||
}
|
||||
|
||||
$ip = $reservation->ip;
|
||||
$reservation = $picked['reservation'];
|
||||
$ipPoolId = $picked['ip_pool_id'];
|
||||
$ip = $reservation->ip;
|
||||
$gateway = (string) $picked['gateway'];
|
||||
$cidr = (string) $picked['cidr'];
|
||||
|
||||
// [5] Assign IP into VM guest OS via PS1 script
|
||||
// [4] Assign IP into VM guest OS via PS1 script (Linux uses assignIPLinux.ps1)
|
||||
$assignOutput = $ssh->assignIp(
|
||||
vmId: $this->vmId,
|
||||
ip: $ip,
|
||||
@@ -136,18 +143,18 @@ class AssignIpToGuestJob implements ShouldQueue
|
||||
guestUser: $guestUser,
|
||||
);
|
||||
|
||||
// [6] If assign failed → release and try next IP
|
||||
// [5] If assign failed → release and try next IP
|
||||
if (!$this->isSuccess($assignOutput)) {
|
||||
Log::warning('AssignIpToGuestJob: assignIp FAIL', [
|
||||
'ip' => $ip,
|
||||
'attempt' => $attempt,
|
||||
'output' => trim($assignOutput),
|
||||
]);
|
||||
$this->releaseIp($reservation);
|
||||
$this->releaseIp($reservation, $ipPoolId);
|
||||
continue;
|
||||
}
|
||||
|
||||
// [7] Verify IP is active inside guest OS via PS1 script
|
||||
// [6] Verify IP is active inside guest OS via PS1 script
|
||||
$verifyOutput = $ssh->verifyIpInGuest(
|
||||
vmId: $this->vmId,
|
||||
ip: $ip,
|
||||
@@ -159,19 +166,19 @@ class AssignIpToGuestJob implements ShouldQueue
|
||||
guestUser: $guestUser,
|
||||
);
|
||||
|
||||
// [8] If verification failed → release and try next IP
|
||||
// [7] If verification failed → release and try next IP
|
||||
if (!$this->isSuccess($verifyOutput)) {
|
||||
Log::warning('AssignIpToGuestJob: verifyIpInGuest FAIL', [
|
||||
'ip' => $ip,
|
||||
'attempt' => $attempt,
|
||||
'output' => trim($verifyOutput),
|
||||
]);
|
||||
$this->releaseIp($reservation);
|
||||
$this->releaseIp($reservation, $ipPoolId);
|
||||
continue;
|
||||
}
|
||||
|
||||
// [9] SUCCESS — persist confirmed IP to DB
|
||||
$this->persistIp($reservation, $moduleVmware);
|
||||
// [8] SUCCESS — persist confirmed IP to module_vmware + virtual_machines + ip_pools
|
||||
$this->persistIp($reservation, $moduleVmware, $ipPoolId, $serviceId);
|
||||
|
||||
Log::info('AssignIpToGuestJob: IP assigned and verified', [
|
||||
'module_vmware_id' => $this->moduleVmwareId,
|
||||
@@ -180,13 +187,17 @@ class AssignIpToGuestJob implements ShouldQueue
|
||||
'attempt' => $attempt,
|
||||
]);
|
||||
|
||||
// Dispatch credential email now that we have the real IP
|
||||
SendVmCredentialsEmailJob::dispatch($this->moduleVmwareId);
|
||||
// Dispatch password change now that IP is confirmed
|
||||
ChangePasswordViaVsphereJob::dispatch(
|
||||
$this->moduleVmwareId,
|
||||
$this->vmId,
|
||||
$this->osFamily,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// [10] All attempts exhausted
|
||||
// [9] All attempts exhausted
|
||||
Log::error('AssignIpToGuestJob: all attempts failed — marking VM inactive', [
|
||||
'module_vmware_id' => $this->moduleVmwareId,
|
||||
'vm_id' => $this->vmId,
|
||||
@@ -194,9 +205,10 @@ class AssignIpToGuestJob implements ShouldQueue
|
||||
]);
|
||||
|
||||
ModuleVmware::where('id', $this->moduleVmwareId)->update(['status' => 0]);
|
||||
Service::where('vm_id', $this->moduleVmwareId)->update(['status' => 'failed']);
|
||||
|
||||
// Still send credential email — try to get IP from vSphere guest identity
|
||||
// so the customer at least gets server info even if IP assignment failed
|
||||
// Attempt to recover IP from vSphere guest identity for admin visibility only.
|
||||
// Do not send credentials here: password has not been rotated yet.
|
||||
try {
|
||||
$moduleVmware = ModuleVmware::find($this->moduleVmwareId);
|
||||
if ($moduleVmware && !$moduleVmware->ip) {
|
||||
@@ -215,13 +227,8 @@ class AssignIpToGuestJob implements ShouldQueue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SendVmCredentialsEmailJob::dispatch($this->moduleVmwareId);
|
||||
Log::info('AssignIpToGuestJob: dispatched SendVmCredentialsEmailJob after failed attempts', [
|
||||
'module_vmware_id' => $this->moduleVmwareId,
|
||||
]);
|
||||
} catch (\Throwable $e) {
|
||||
Log::error('AssignIpToGuestJob: failed to send email after exhausted attempts', [
|
||||
Log::error('AssignIpToGuestJob: failed to recover IP after exhausted attempts', [
|
||||
'module_vmware_id' => $this->moduleVmwareId,
|
||||
'error' => $e->getMessage(),
|
||||
]);
|
||||
@@ -237,10 +244,11 @@ class AssignIpToGuestJob implements ShouldQueue
|
||||
]);
|
||||
|
||||
ModuleVmware::where('id', $this->moduleVmwareId)->update(['status' => 0]);
|
||||
Service::where('vm_id', $this->moduleVmwareId)->update(['status' => 'failed']);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// assignIp / verifyIp helpers
|
||||
// IP reservation helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
@@ -253,14 +261,15 @@ class AssignIpToGuestJob implements ShouldQueue
|
||||
*
|
||||
* Returns the pending ModuleIpAddress record, or null if pool is exhausted.
|
||||
*/
|
||||
/**
|
||||
* @return array{reservation:ModuleIpAddress,ip_pool_id:int,gateway:string,cidr:int}|null
|
||||
*/
|
||||
private function reserveIp(
|
||||
int $networkId,
|
||||
int $hostId,
|
||||
string $gateway,
|
||||
string $netmask,
|
||||
string $dns,
|
||||
): ?ModuleIpAddress {
|
||||
return DB::transaction(function () use ($networkId, $hostId, $gateway, $netmask, $dns) {
|
||||
): ?array {
|
||||
return DB::transaction(function () use ($networkId, $hostId, $dns) {
|
||||
// Lock existing rows to prevent another job from picking the same IP
|
||||
$usedIps = ModuleIpAddress::where('module_network_id', $networkId)
|
||||
->lockForUpdate()
|
||||
@@ -268,6 +277,9 @@ class AssignIpToGuestJob implements ShouldQueue
|
||||
->toArray();
|
||||
|
||||
$ipPool = IpPool::where('module_network_id', $networkId)
|
||||
->where('status', 'free')
|
||||
->whereNotNull('gateway')
|
||||
->where('gateway', '!=', '')
|
||||
->whereNotIn('ip_address', $usedIps)
|
||||
->lockForUpdate()
|
||||
->inRandomOrder()
|
||||
@@ -277,8 +289,15 @@ class AssignIpToGuestJob implements ShouldQueue
|
||||
return null;
|
||||
}
|
||||
|
||||
// Hold the pool slot
|
||||
$ipPool->update(['status' => 'reserved']);
|
||||
|
||||
$cidr = (int) ($ipPool->cidr ?: 24);
|
||||
$gateway = (string) ($ipPool->gateway ?? '');
|
||||
$netmask = $this->cidrToNetmask($cidr);
|
||||
|
||||
// Create a pending reservation — acts as a distributed lock on this IP
|
||||
return ModuleIpAddress::create([
|
||||
$reservation = ModuleIpAddress::create([
|
||||
'vm_name' => '__pending__',
|
||||
'module_host_id' => $hostId,
|
||||
'module_network_id' => $networkId,
|
||||
@@ -288,6 +307,13 @@ class AssignIpToGuestJob implements ShouldQueue
|
||||
'dns' => $dns,
|
||||
'status' => 0, // 0 = pending/reserved, not yet confirmed
|
||||
]);
|
||||
|
||||
return [
|
||||
'reservation' => $reservation,
|
||||
'ip_pool_id' => $ipPool->id,
|
||||
'gateway' => $gateway,
|
||||
'cidr' => $cidr,
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
@@ -295,10 +321,13 @@ class AssignIpToGuestJob implements ShouldQueue
|
||||
* Release a failed reservation by deleting the pending ModuleIpAddress row.
|
||||
* This returns the IP to the available pool.
|
||||
*/
|
||||
private function releaseIp(ModuleIpAddress $reservation): void
|
||||
private function releaseIp(ModuleIpAddress $reservation, ?int $ipPoolId): void
|
||||
{
|
||||
try {
|
||||
$reservation->delete();
|
||||
if ($ipPoolId) {
|
||||
IpPool::where('id', $ipPoolId)->update(['status' => 'free']);
|
||||
}
|
||||
Log::info('AssignIpToGuestJob: IP reservation released', ['ip' => $reservation->ip]);
|
||||
} catch (\Throwable $e) {
|
||||
Log::warning('AssignIpToGuestJob: could not release IP reservation', [
|
||||
@@ -312,12 +341,17 @@ class AssignIpToGuestJob implements ShouldQueue
|
||||
* Confirm a successful IP assignment:
|
||||
* - Update ModuleIpAddress: status → 1 (assigned), vm_name → real name
|
||||
* - Update ModuleVmware: ip, dns, module_ip_address FK
|
||||
* - Update VirtualMachines: ip, ip_pool_id FK (matched by vmId)
|
||||
*
|
||||
* IP is only written to DB AFTER guest-side verification passes.
|
||||
*/
|
||||
private function persistIp(ModuleIpAddress $reservation, ModuleVmware $moduleVmware): void
|
||||
{
|
||||
DB::transaction(function () use ($reservation, $moduleVmware) {
|
||||
private function persistIp(
|
||||
ModuleIpAddress $reservation,
|
||||
ModuleVmware $moduleVmware,
|
||||
?int $ipPoolId,
|
||||
?int $serviceId,
|
||||
): void {
|
||||
DB::transaction(function () use ($reservation, $moduleVmware, $ipPoolId, $serviceId) {
|
||||
$reservation->update([
|
||||
'vm_name' => $moduleVmware->name ?? ('VM-' . $this->vmId),
|
||||
'status' => 1, // assigned
|
||||
@@ -328,9 +362,38 @@ class AssignIpToGuestJob implements ShouldQueue
|
||||
'dns' => $reservation->dns,
|
||||
'module_ip_address' => $reservation->id,
|
||||
]);
|
||||
|
||||
// Sync to virtual_machines — matched by vSphere vmId
|
||||
VirtualMachines::where('vm_id', $this->vmId)->update([
|
||||
'ip' => $reservation->ip,
|
||||
'ip_pool_id' => $ipPoolId,
|
||||
]);
|
||||
|
||||
// Flip ip_pools row to assigned and link service
|
||||
if ($ipPoolId) {
|
||||
IpPool::where('id', $ipPoolId)->update([
|
||||
'status' => 'assigned',
|
||||
'service_id' => $serviceId,
|
||||
]);
|
||||
}
|
||||
|
||||
// Touch services row so server_name reflects the provisioned VM.
|
||||
// Note: services.vm_id is an FK to module_vmware.id (already set by VmServiceSyncService) — do not overwrite with vSphere vmId string.
|
||||
if ($serviceId) {
|
||||
Service::where('id', $serviceId)->update([
|
||||
'server_name' => $moduleVmware->name,
|
||||
]);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** Convert CIDR prefix length to dotted-decimal netmask (e.g. 24 → 255.255.255.0). */
|
||||
private function cidrToNetmask(int $cidr): string
|
||||
{
|
||||
$cidr = max(0, min(32, $cidr));
|
||||
return long2ip(-1 << (32 - $cidr));
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Micro-helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
@@ -338,6 +401,10 @@ class AssignIpToGuestJob implements ShouldQueue
|
||||
/** PS1 scripts return JSON: {"status":"ok"} or {"status":"error","message":"..."} */
|
||||
private function isSuccess(string $output): bool
|
||||
{
|
||||
if (str_contains(strtoupper(trim($output)), 'SUCCESS')) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$data = json_decode(trim($output), true);
|
||||
return is_array($data) && ($data['status'] ?? '') === 'ok';
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ namespace App\Jobs\VirtualMachine\Provision;
|
||||
|
||||
use App\Http\Facade\SignRSA;
|
||||
use App\Models\ModuleVmware;
|
||||
use App\Models\Service;
|
||||
use App\Models\vCenter;
|
||||
use App\Services\VmProvisionService;
|
||||
use App\Services\VsphereHostSshService;
|
||||
@@ -22,15 +23,15 @@ use Illuminate\Support\Facades\Log;
|
||||
* Resets the VM guest OS password via:
|
||||
* Laravel → SSH (10.10.10.173) → PowerShell PS1 → Invoke-VMScript (guest OS)
|
||||
*
|
||||
* Flow:
|
||||
* [1] VM already exists, vm_id known
|
||||
* Dispatched by AssignIpToGuestJob after IP is confirmed. Flow:
|
||||
* [1] VM already exists, vm_id known, IP already assigned
|
||||
* [2] Generate random 16-char password (plain text)
|
||||
* [3] Load vCenter credentials from module_vcenter
|
||||
* [4] SSH into vSphere host
|
||||
* [5] Execute changePassword{Linux|Windows}.ps1 with 5 params
|
||||
* [6] Parse "SUCCESS" / "FAIL" from script output
|
||||
* [7] On SUCCESS → save encrypted password to DB
|
||||
* [8] Dispatch AssignIpToGuestJob with the new guest password
|
||||
* [8] Dispatch SendVmCredentialsEmailJob (IP + password both ready)
|
||||
* [9] On FAIL → throw exception → job retries
|
||||
*/
|
||||
class ChangePasswordViaVsphereJob implements ShouldQueue
|
||||
@@ -84,10 +85,13 @@ class ChangePasswordViaVsphereJob implements ShouldQueue
|
||||
// [6] Parse output — throw on FAIL so job retries
|
||||
$this->assertSuccess($output);
|
||||
|
||||
// [7] Save encrypted + plain password (IP will be set later by AssignIpToGuestJob)
|
||||
// [7] Save encrypted + plain password (IP already assigned by AssignIpToGuestJob)
|
||||
$provision->updatePassword($this->moduleVmwareId, $newPassword);
|
||||
|
||||
// [8] Send VM credentials email (AssignIpToGuestJob bypassed temporarily)
|
||||
// Provisioning complete — activate the service record
|
||||
Service::where('vm_id', $this->moduleVmwareId)->update(['status' => 'active']);
|
||||
|
||||
// [8] Send credentials email — IP + new password are both ready at this point
|
||||
SendVmCredentialsEmailJob::dispatch($this->moduleVmwareId);
|
||||
|
||||
Log::info('ChangePasswordViaVsphereJob: completed', [
|
||||
@@ -107,6 +111,7 @@ class ChangePasswordViaVsphereJob implements ShouldQueue
|
||||
]);
|
||||
|
||||
ModuleVmware::where('id', $this->moduleVmwareId)->update(['status' => 0]);
|
||||
Service::where('vm_id', $this->moduleVmwareId)->update(['status' => 'failed']);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -102,9 +102,7 @@ class WaitVmReadyJob implements ShouldQueue
|
||||
ChangePasswordViaVsphereJob::dispatch(
|
||||
$this->moduleVmwareId,
|
||||
$this->vmId,
|
||||
$guestIp,
|
||||
$this->osFamily,
|
||||
$this->username,
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user