Add VM rebuild pipeline with isolated job chain and admin log API

This commit is contained in:
baodev0
2026-05-29 18:19:38 +07:00
parent 7ae237ee9d
commit 9c68273b4a
27 changed files with 2391 additions and 199 deletions
@@ -0,0 +1,81 @@
<?php
declare(strict_types=1);
namespace App\Jobs\VirtualMachine\Rebuild;
use App\Services\VirtualMachine\Rebuild\VmRebuildAddonService;
use App\Services\VirtualMachine\Rebuild\VmRebuildService;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Throwable;
/**
* RebuildAddonInstallJob
*
* Reapplies CPU / RAM / Disk addons attached to the existing service onto
* the new VM. Idempotent — Upgrade* primitives are no-ops when the spec
* already matches.
*/
final class RebuildAddonInstallJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public int $tries = 3;
public int $timeout = 300;
public int $backoff = 30;
public function __construct(
public readonly int $moduleVmwareId,
public readonly int $newOsId,
public readonly string $newVmId,
public readonly int $reinstallLogId,
) {
}
public function handle(
VmRebuildService $rebuildService,
VmRebuildAddonService $addonService,
): void {
Log::info('RebuildAddonInstallJob: start', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
]);
$ctx = $rebuildService
->buildContext($this->moduleVmwareId, $this->newOsId, $this->newVmId);
$addonService->reinstall($ctx);
RebuildFinalizeJob::dispatch(
$this->moduleVmwareId,
$this->newOsId,
$this->newVmId,
$this->reinstallLogId,
)->onQueue('default');
Log::info('RebuildAddonInstallJob: completed, finalize dispatched', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
]);
}
public function failed(Throwable $e): void
{
Log::error('RebuildAddonInstallJob: failed — old VM preserved', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
'error' => $e->getMessage(),
]);
\App\Models\ReinstallLog::where('id', $this->reinstallLogId)->update([
'status' => \App\Models\ReinstallLog::STATUS_FAILED,
'error_message' => 'AddonInstall failed: ' . $e->getMessage(),
'updated_at' => now(),
]);
}
}
@@ -0,0 +1,92 @@
<?php
declare(strict_types=1);
namespace App\Jobs\VirtualMachine\Rebuild;
use App\Services\VirtualMachine\Rebuild\VmRebuildIpService;
use App\Services\VirtualMachine\Rebuild\VmRebuildService;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Throwable;
/**
* RebuildAssignIpJob
*
* Reuses the existing service IP on the new VM (no allocation, no pool
* mutation, no ownership change). Idempotent: re-running it simply runs the
* assign + verify scripts again with the same parameters.
*/
final class RebuildAssignIpJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public int $tries = 3;
public int $timeout = 600;
public int $backoff = 30;
public function __construct(
public readonly int $moduleVmwareId,
public readonly int $newOsId,
public readonly string $newVmId,
public readonly int $reinstallLogId,
) {
}
public function handle(
VmRebuildService $rebuildService,
VmRebuildIpService $ipService,
): void {
Log::info('RebuildAssignIpJob: start', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
'attempt' => $this->attempts(),
]);
$ctx = $rebuildService
->buildContext($this->moduleVmwareId, $this->newOsId, $this->newVmId);
try {
$ipService->assignAndVerify($ctx);
} catch (Throwable $e) {
// Surface the actual PS1 output into the reinstall log so admin
// can see WHY it failed without diving into laravel.log.
\App\Models\ReinstallLog::where('id', $this->reinstallLogId)->update([
'error_message' => 'AssignIp attempt ' . $this->attempts() . ': ' . $e->getMessage(),
'updated_at' => now(),
]);
throw $e;
}
RebuildChangePasswordJob::dispatch(
$this->moduleVmwareId,
$this->newOsId,
$this->newVmId,
$this->reinstallLogId,
)->onQueue('default');
Log::info('RebuildAssignIpJob: completed, password job dispatched', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
]);
}
public function failed(Throwable $e): void
{
Log::error('RebuildAssignIpJob: all retries exhausted — old VM preserved', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
'error' => $e->getMessage(),
]);
\App\Models\ReinstallLog::where('id', $this->reinstallLogId)->update([
'status' => \App\Models\ReinstallLog::STATUS_FAILED,
'error_message' => 'AssignIp failed: ' . $e->getMessage(),
'updated_at' => now(),
]);
}
}
@@ -0,0 +1,81 @@
<?php
declare(strict_types=1);
namespace App\Jobs\VirtualMachine\Rebuild;
use App\Services\VirtualMachine\Rebuild\VmRebuildPasswordService;
use App\Services\VirtualMachine\Rebuild\VmRebuildService;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Throwable;
/**
* RebuildChangePasswordJob
*
* Generates a fresh credential and applies it to the new VM via the shared
* VsphereHostSshService. Saves the encrypted + plaintext copy onto the
* existing ModuleVmware row.
*/
final class RebuildChangePasswordJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public int $tries = 5;
public int $timeout = 180;
public int $backoff = 60;
public function __construct(
public readonly int $moduleVmwareId,
public readonly int $newOsId,
public readonly string $newVmId,
public readonly int $reinstallLogId,
) {
}
public function handle(
VmRebuildService $rebuildService,
VmRebuildPasswordService $passwordService,
): void {
Log::info('RebuildChangePasswordJob: start', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
]);
$ctx = $rebuildService
->buildContext($this->moduleVmwareId, $this->newOsId, $this->newVmId);
$passwordService->rotate($ctx, $this->reinstallLogId);
RebuildAddonInstallJob::dispatch(
$this->moduleVmwareId,
$this->newOsId,
$this->newVmId,
$this->reinstallLogId,
)->onQueue('default');
Log::info('RebuildChangePasswordJob: completed, addon job dispatched', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
]);
}
public function failed(Throwable $e): void
{
Log::error('RebuildChangePasswordJob: failed — old VM preserved', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
'error' => $e->getMessage(),
]);
\App\Models\ReinstallLog::where('id', $this->reinstallLogId)->update([
'status' => \App\Models\ReinstallLog::STATUS_FAILED,
'error_message' => 'ChangePassword failed: ' . $e->getMessage(),
'updated_at' => now(),
]);
}
}
@@ -0,0 +1,103 @@
<?php
declare(strict_types=1);
namespace App\Jobs\VirtualMachine\Rebuild;
use App\Http\Services\vCenter as VCenterService;
use App\Models\ModuleVmware;
use App\Models\vCenter as VCenterModel;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Throwable;
/**
* RebuildDeleteOldVmJob
*
* Final step of the rebuild pipeline. Runs only AFTER the DB has already
* been swapped to point at the new VM, so even if this job fails the
* customer's service is fully online on the new VM — the old VM just
* becomes an orphan that an operator can clean up later.
*
* Defensive guard: before deleting, re-read ModuleVmware.vmware_id from DB
* and refuse to delete if it still matches the "old" id we were told about.
* That would mean the swap somehow rolled back, and deleting would destroy
* the live VM.
*/
final class RebuildDeleteOldVmJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public int $tries = 3;
public int $timeout = 180;
public int $backoff = 60;
public function __construct(
public readonly int $moduleVmwareId,
public readonly string $oldVmId,
public readonly ?int $reinstallLogId = null,
) {
}
public function handle(VCenterService $vCenter): void
{
if ($this->oldVmId === '') {
Log::info('RebuildDeleteOldVmJob: nothing to delete (oldVmId empty)');
return;
}
$vm = ModuleVmware::find($this->moduleVmwareId);
if (!$vm) {
Log::warning('RebuildDeleteOldVmJob: ModuleVmware vanished, skipping', [
'module_vmware_id' => $this->moduleVmwareId,
]);
return;
}
if ((string) $vm->vmware_id === $this->oldVmId) {
// DB swap never happened — refuse to delete the live VM.
Log::error('RebuildDeleteOldVmJob: ABORT — module_vmware still references oldVmId', [
'module_vmware_id' => $this->moduleVmwareId,
'old_vm_id' => $this->oldVmId,
]);
return;
}
$vcenter = VCenterModel::find($vm->vcenter_id);
if (!$vcenter || !$vCenter->loginServer($vcenter)) {
Log::error('RebuildDeleteOldVmJob: cannot login to vCenter', [
'old_vm_id' => $this->oldVmId,
]);
return;
}
try {
$vCenter->PowerOffVirtualMachine($this->oldVmId);
$res = $vCenter->DeleteVirtualMachine($this->oldVmId);
Log::info('RebuildDeleteOldVmJob: old VM deleted', [
'old_vm_id' => $this->oldVmId,
'result' => $res,
]);
} catch (Throwable $e) {
Log::error('RebuildDeleteOldVmJob: delete failed', [
'old_vm_id' => $this->oldVmId,
'error' => $e->getMessage(),
]);
// Re-throw so Laravel retries.
throw $e;
}
}
public function failed(Throwable $e): void
{
Log::error('RebuildDeleteOldVmJob: retries exhausted — manual cleanup required', [
'module_vmware_id' => $this->moduleVmwareId,
'old_vm_id' => $this->oldVmId,
'error' => $e->getMessage(),
]);
}
}
@@ -0,0 +1,184 @@
<?php
declare(strict_types=1);
namespace App\Jobs\VirtualMachine\Rebuild;
use App\Jobs\Mail\Admin\AdminOrderNotificationJob;
use App\Jobs\Mail\SendInformationVirtualMachine;
use App\Models\Invoice;
use App\Models\ModuleVmware;
use App\Models\Service;
use App\Models\ReinstallLog;
use App\Models\Setting;
use App\Services\VirtualMachine\Rebuild\VmRebuildService;
use App\Services\VirtualMachine\Rebuild\VmRebuildSwapService;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Throwable;
/**
* RebuildFinalizeJob
*
* Final validation + DB swap + credential email. Only after this step is the
* old VM safe to delete.
*
* [1] Health check the new VM in vCenter.
* [2] Swap module_vmware.vmware_id (+ virtual_machines.vm_id) atomically.
* [3] Send credentials email + admin notification.
* [4] Dispatch RebuildDeleteOldVmJob — deletion happens last.
*
* If health check fails: nothing is swapped, old VM stays attached to the
* service, and the operator gets a failed-job alert.
*/
final class RebuildFinalizeJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public int $tries = 3;
public int $timeout = 300;
public int $backoff = 30;
public function __construct(
public readonly int $moduleVmwareId,
public readonly int $newOsId,
public readonly string $newVmId,
public readonly int $reinstallLogId,
) {
}
public function handle(
VmRebuildService $rebuildService,
VmRebuildSwapService $swapService,
): void {
Log::info('RebuildFinalizeJob: start', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
]);
$ctx = $rebuildService
->buildContext($this->moduleVmwareId, $this->newOsId, $this->newVmId);
// [1] Health check before touching DB.
$swapService->healthCheck($ctx);
$oldVmId = $ctx->oldVmId;
// [2] Atomic DB swap.
$swapService->swapDatabaseReferences($ctx);
// [3] Notify customer with new credentials.
$this->sendCredentialsEmail($this->moduleVmwareId);
ReinstallLog::where('id', $this->reinstallLogId)->update([
'status' => ReinstallLog::STATUS_COMPLETED,
'new_vmware_id' => $ctx->newVmId,
'note' => 'Rebuild OK',
'completed_at' => now(),
'updated_at' => now(),
]);
// [4] Schedule old-VM deletion — last step, after everything is committed.
RebuildDeleteOldVmJob::dispatch(
$this->moduleVmwareId,
$oldVmId,
$this->reinstallLogId,
)->onQueue('default');
Log::info('RebuildFinalizeJob: completed', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
'old_vm_id' => $oldVmId,
]);
}
public function failed(Throwable $e): void
{
Log::error('RebuildFinalizeJob: failed — DB swap aborted, old VM remains attached', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
'error' => $e->getMessage(),
]);
ReinstallLog::where('id', $this->reinstallLogId)->update([
'status' => ReinstallLog::STATUS_FAILED,
'error_message' => $e->getMessage(),
'updated_at' => now(),
]);
}
private function sendCredentialsEmail(int $moduleVmwareId): void
{
$vm = ModuleVmware::with(['customer', 'os.osGroup', 'order.orderItems.service'])
->find($moduleVmwareId);
if (!$vm || !$vm->customer?->email) {
Log::warning('RebuildFinalizeJob: skip credentials email — VM/customer missing', [
'module_vmware_id' => $moduleVmwareId,
]);
return;
}
$setting = Setting::where('key', 'setting')->first();
$settingValue = $setting ? json_decode((string) $setting->value) : null;
$orderItem = $vm->order?->orderItems?->first();
$months = (int) ($orderItem?->months ?? 0);
$osGroup = $vm->os?->osGroup?->name ?? '';
$isWindows = str_contains(strtolower($osGroup), 'windows');
$osUsername = $isWindows ? 'Administrator' : 'root';
$emailData = [
'company_name' => $settingValue->company ?? 'Thế Giới Số',
'company_logo' => $settingValue->logo ?? '',
'client_name' => $vm->customer->fullname ?? '',
'product_name' => $orderItem?->getProductAttribute()?->name ?? 'Cloud VPS',
'product_billing' => $this->billingLabel($months),
'product_price' => $orderItem ? number_format((float) $orderItem->price, 0, ',', '.') : 'N/A',
'service_host' => $vm->ip ?? 'N/A',
'service_username' => $osUsername,
'service_port' => (string) ($vm->port ?? '22'),
'service_password' => $vm->plain_password ?? '(xem tai khoan quan ly)',
'os_name' => $vm->os ? ($osGroup . ' ' . $vm->os->version) : 'Unknown OS',
'service_due_exp' => $vm->next_due_date
? date('d/m/Y', (int) $vm->next_due_date)
: 'N/A',
'is_rebuild' => true,
];
$invoice = Invoice::where('order_id', $vm->order_id)->first();
SendInformationVirtualMachine::dispatch($emailData, $vm->customer, $invoice?->id);
AdminOrderNotificationJob::dispatch([
'customer_name' => $vm->customer->fullname ?? '',
'customer_email' => $vm->customer->email,
'client_ip' => null,
'order_num' => $vm->order?->order_num ?? $vm->order?->id ?? 'N/A',
'product_name' => ($orderItem?->getProductAttribute()?->name ?? 'Cloud VPS') . ' (Rebuild)',
'product_type' => 'Cloud VPS Rebuild',
'billing_cycle' => $this->billingLabel($months),
'total' => $orderItem?->price ?? 0,
'created_at' => now()->format('d/m/Y H:i:s'),
'ip' => $vm->ip ?? '',
'username' => $osUsername,
'password' => $vm->plain_password ?? '',
]);
}
private function billingLabel(int $months): string
{
return match ($months) {
1 => 'Hang thang',
3 => '3 Thang',
6 => '6 Thang',
12 => 'Hang nam',
24 => '2 Nam',
default => $months . ' Thang',
};
}
}
@@ -0,0 +1,129 @@
<?php
declare(strict_types=1);
namespace App\Jobs\VirtualMachine\Rebuild;
use App\Models\ModuleVmware;
use App\Models\ReinstallLog;
use App\Models\Service;
use App\Services\VirtualMachine\Rebuild\VmRebuildCloneService;
use App\Services\VirtualMachine\Rebuild\VmRebuildService;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Throwable;
/**
* RebuildVirtualMachineJob
*
* First step of the isolated rebuild pipeline.
*
* [1] Build RebuildContext from current DB state (idempotent reload).
* [2] Power off old VM (best-effort).
* [3] Clone new VM from chosen OS template.
* [4] Power on new VM + wait for VMware Tools.
* [5] Dispatch RebuildAssignIpJob with the new vSphere vm_id.
*
* On failure: old VM remains active, no DB references are swapped, no IP
* change, no deletion. ReinstallLog is marked failed; service is reverted
* to 'active' for visibility.
*/
final class RebuildVirtualMachineJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public int $tries = 1;
public int $timeout = 900;
public function __construct(
public readonly int $moduleVmwareId,
public readonly int $newOsId,
public readonly int $reinstallLogId,
) {
}
public function handle(
VmRebuildService $rebuildService,
VmRebuildCloneService $cloneService,
): void {
Log::info('RebuildVirtualMachineJob: start', [
'module_vmware_id' => $this->moduleVmwareId,
'new_os_id' => $this->newOsId,
'reinstall_log_id' => $this->reinstallLogId,
]);
try {
$ctx = $rebuildService->buildContext($this->moduleVmwareId, $this->newOsId);
$this->markServiceStatus($ctx->serviceId, 'rebuilding');
$cloneService->powerOffOldVm($ctx);
$newVmId = $cloneService->clone($ctx);
$ctx = $ctx->withNewVmId($newVmId);
$cloneService->powerOnAndWaitTools($ctx);
$this->markLog([
'status' => ReinstallLog::STATUS_IN_PROGRESS,
'new_vmware_id' => $newVmId,
'started_at' => now(),
]);
RebuildAssignIpJob::dispatch(
$this->moduleVmwareId,
$this->newOsId,
$newVmId,
$this->reinstallLogId,
)->onQueue('default');
Log::info('RebuildVirtualMachineJob: completed, IP job dispatched', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $newVmId,
]);
} catch (Throwable $e) {
$this->onFailure($e);
throw $e;
}
}
public function failed(Throwable $e): void
{
$this->onFailure($e);
}
private function onFailure(Throwable $e): void
{
Log::error('RebuildVirtualMachineJob: failed — old VM preserved', [
'module_vmware_id' => $this->moduleVmwareId,
'error' => $e->getMessage(),
]);
ModuleVmware::where('id', $this->moduleVmwareId)->update(['status' => 1]);
$service = Service::where('vm_id', $this->moduleVmwareId)->first();
if ($service) {
$service->update(['status' => 'active']);
}
$this->markLog([
'status' => ReinstallLog::STATUS_FAILED,
'error_message' => $e->getMessage(),
]);
}
private function markServiceStatus(?int $serviceId, string $status): void
{
if ($serviceId) {
Service::where('id', $serviceId)->update(['status' => $status]);
}
}
private function markLog(array $attrs): void
{
ReinstallLog::where('id', $this->reinstallLogId)->update($attrs + ['updated_at' => now()]);
}
}