Add VM rebuild pipeline with isolated job chain and admin log API

This commit is contained in:
baodev0
2026-05-29 18:19:38 +07:00
parent 7ae237ee9d
commit 9c68273b4a
27 changed files with 2391 additions and 199 deletions
+2 -1
View File
@@ -32,7 +32,8 @@
"Bash(redis-cli --no-auth-warning llen \"the_gioi_so_horizon:queue:default\")",
"Bash(redis-cli --no-auth-warning lrange \"the_gioi_so_horizon:queue:default\" 0 4)",
"Bash(redis-cli *)",
"Bash(echo \"exit=$?\")"
"Bash(echo \"exit=$?\")",
"Read(//Users/baro/Desktop/Workspace/thegioiso/web/portal-infra/portal-client/app/\\(main\\)/dich-vu/**)"
]
}
}
@@ -0,0 +1,81 @@
<?php
declare(strict_types=1);
namespace App\DTOs\VirtualMachine\Rebuild;
/**
* Immutable context object carried through the rebuild pipeline.
*
* Built fresh by VmRebuildService::buildContext() at the start of every
* rebuild job so it always reflects the current DB state. Jobs themselves
* only serialize a few primitive identifiers — they reload context from
* DB to stay idempotent.
*/
final class RebuildContext
{
/**
* @param array{cpu:int,ram:int,disk:int} $addonTotals
*/
public function __construct(
public readonly int $moduleVmwareId,
public readonly ?int $serviceId,
public readonly int $customerId,
public readonly int $newOsId,
public readonly string $oldVmId,
public readonly ?string $newVmId,
public readonly string $vmName,
public readonly string $transientName,
public readonly string $ip,
public readonly string $gateway,
public readonly string $netmask,
public readonly int $cidr,
public readonly string $dns,
public readonly ?int $moduleIpAddressId,
public readonly int $vcenterId,
public readonly int $hostId,
public readonly bool $isWindows,
public readonly string $osFamily,
public readonly string $guestUser,
public readonly mixed $template,
public readonly mixed $datacenterRef,
public readonly mixed $hostRef,
public readonly int $baseCpu,
public readonly int $baseRam,
public readonly int $baseDisk,
public readonly array $addonTotals,
) {
}
public function withNewVmId(string $newVmId): self
{
return new self(
$this->moduleVmwareId,
$this->serviceId,
$this->customerId,
$this->newOsId,
$this->oldVmId,
$newVmId,
$this->vmName,
$this->transientName,
$this->ip,
$this->gateway,
$this->netmask,
$this->cidr,
$this->dns,
$this->moduleIpAddressId,
$this->vcenterId,
$this->hostId,
$this->isWindows,
$this->osFamily,
$this->guestUser,
$this->template,
$this->datacenterRef,
$this->hostRef,
$this->baseCpu,
$this->baseRam,
$this->baseDisk,
$this->addonTotals,
);
}
}
@@ -0,0 +1,179 @@
<?php
declare(strict_types=1);
namespace App\Http\Controllers\Api\Admin\Rebuild;
use App\Http\Controllers\Controller;
use App\Models\ReinstallLog;
use Illuminate\Http\JsonResponse;
use Illuminate\Http\Request;
/**
* AdminReinstallLogController
*
* Admin-facing read API for VM rebuild history. Surfaces:
* - which customer initiated each rebuild
* - which service / VM was rebuilt
* - which OS was chosen
* - status lifecycle + timestamps
* - the credential that was applied to the rebuilt VM (decrypted on detail)
*
* Routes:
* GET /v1/administrator/rebuild-logs → paginated index
* GET /v1/administrator/rebuild-logs/stats → counts by status
* GET /v1/administrator/rebuild-logs/{id} → full detail (incl. password)
*/
final class AdminReinstallLogController extends Controller
{
public function index(Request $request): JsonResponse
{
$perPage = min((int) $request->get('per_page', 20), 200);
$query = ReinstallLog::query()
->with([
'customer:id,fullname,email',
'service:id,uuid,server_name,status',
'vm:id,name,vmware_id,ip',
'os:id,version,os_group_id',
'os.osGroup:id,name',
])
->orderByDesc('id');
if ($status = $request->query('status')) {
$query->where('status', $status);
}
if ($customerId = $request->query('customer_id')) {
$query->where('custome_id', (int) $customerId);
}
if ($serviceId = $request->query('service_id')) {
$query->where('service_id', (int) $serviceId);
}
if ($osId = $request->query('os_id')) {
$query->where('os_id', (int) $osId);
}
if ($initiator = $request->query('initiator')) {
$query->where('initiator', $initiator);
}
if ($from = $request->query('date_from')) {
$query->whereDate('created_at', '>=', $from);
}
if ($to = $request->query('date_to')) {
$query->whereDate('created_at', '<=', $to);
}
if ($search = trim((string) $request->query('search', ''))) {
$query->where(function ($q) use ($search) {
$q->where('ip_address', 'like', "%{$search}%")
->orWhere('old_vmware_id', 'like', "%{$search}%")
->orWhere('new_vmware_id', 'like', "%{$search}%")
->orWhereHas('customer', function ($cq) use ($search) {
$cq->where('email', 'like', "%{$search}%")
->orWhere('fullname', 'like', "%{$search}%");
})
->orWhereHas('vm', fn ($vq) => $vq->where('name', 'like', "%{$search}%"));
});
}
$paginator = $query->paginate($perPage);
return response()->json([
'error' => false,
'data' => [
'items' => collect($paginator->items())->map(fn (ReinstallLog $log) => $this->summarize($log))->values(),
'total' => $paginator->total(),
'per_page' => $paginator->perPage(),
'current_page' => $paginator->currentPage(),
'last_page' => $paginator->lastPage(),
],
]);
}
public function show(int $id): JsonResponse
{
$log = ReinstallLog::with([
'customer:id,fullname,email,phone',
'service:id,uuid,server_name,status,product_id',
'service.product:id,name',
'vm:id,name,vmware_id,ip,port,user,os_id,customer_id',
'os:id,version,os_group_id,template_id',
'os.osGroup:id,name,thumbnail',
])->find($id);
if (!$log) {
return response()->json(['error' => true, 'message' => 'Không tìm thấy log rebuild'], 404);
}
$data = $this->summarize($log);
$data['password'] = $log->decryptedPassword();
$data['error_message'] = $log->error_message;
$data['vm'] = $log->vm ? [
'id' => $log->vm->id,
'name' => $log->vm->name,
'ip' => $log->vm->ip,
'port' => $log->vm->port,
'user' => $log->vm->user,
'vmware_id' => $log->vm->vmware_id,
] : null;
return response()->json([
'error' => false,
'data' => $data,
]);
}
public function stats(): JsonResponse
{
return response()->json([
'error' => false,
'data' => [
'total' => ReinstallLog::count(),
'today' => ReinstallLog::whereDate('created_at', today())->count(),
'by_status' => ReinstallLog::selectRaw('status, COUNT(*) as count')
->groupBy('status')->pluck('count', 'status'),
'by_initiator' => ReinstallLog::selectRaw('initiator, COUNT(*) as count')
->groupBy('initiator')->pluck('count', 'initiator'),
],
]);
}
/**
* Compact list-row projection. Password is intentionally omitted here —
* detail endpoint is where the decrypted credential is returned.
*/
private function summarize(ReinstallLog $log): array
{
return [
'id' => $log->id,
'status' => $log->status,
'initiator' => $log->initiator,
'note' => $log->note,
'ip_address' => $log->ip_address,
'old_vmware_id' => $log->old_vmware_id,
'new_vmware_id' => $log->new_vmware_id,
'password_supplied_by_customer' => (bool) $log->password_supplied_by_customer,
'has_credentials' => $log->password !== null || $log->plain_password !== null,
'created_at' => $log->created_at,
'started_at' => $log->started_at,
'completed_at' => $log->completed_at,
'customer' => $log->customer ? [
'id' => $log->customer->id,
'fullname' => $log->customer->fullname,
'email' => $log->customer->email,
] : null,
'service' => $log->service ? [
'id' => $log->service->id,
'uuid' => $log->service->uuid,
'server_name' => $log->service->server_name,
'status' => $log->service->status,
] : null,
'os' => $log->os ? [
'id' => $log->os->id,
'version' => $log->os->version,
'group' => $log->os->osGroup ? [
'id' => $log->os->osGroup->id,
'name' => $log->os->osGroup->name,
] : null,
] : null,
];
}
}
@@ -11,9 +11,12 @@ use App\Models\Service;
use App\Repositories\Api\ServiceClient\ServiceClientRepository;
use App\Services\ActivityLogger;
use App\Services\Api\ServiceClient\ServiceClientService;
use App\Services\VirtualMachine\Rebuild\VmRebuildService;
use Illuminate\Http\JsonResponse;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Throwable;
class ServiceController extends Controller
{
@@ -290,12 +293,12 @@ class ServiceController extends Controller
]);
}
public function rebuild(Request $request, string $uuid, vCenter $serviceVCenter): JsonResponse
public function rebuild(Request $request, string $uuid, VmRebuildService $rebuildService): JsonResponse
{
$customer = $request->user();
$payload = $request->validate([
'os_id' => ['required', 'integer', 'exists:os,id'],
'password' => ['nullable', 'string', 'max:255'],
'password' => ['nullable', 'string', 'min:8', 'max:255'],
]);
$service = Service::with(['product.osVersions', 'vmware'])
@@ -330,27 +333,26 @@ class ServiceController extends Controller
return response()->json(['success' => false, 'message' => 'Dịch vụ chưa được cấp phát VM'], 404);
}
DB::table('reinstall_logs')->insert([
'product_id' => $vm->id,
'product_type' => 4,
'custome_id' => $customer->id,
'os_id' => $payload['os_id'],
'status' => 'pending',
'initiator' => 'client',
'note' => 'Client rebuild via service UUID: ' . $uuid,
'created_at' => now(),
'updated_at' => now(),
]);
$result = $serviceVCenter->reBuildVPS((string) $vm->name);
if (is_string($result) || ($result['error'] ?? false)) {
return response()->json(
[
'success' => false,
'message' => is_string($result) ? $result : $result['message'] ?? 'Không thể rebuild VM',
],
400,
try {
$log = $rebuildService->dispatchRebuild(
moduleVmwareId: (int) $vm->id,
newOsId: (int) $payload['os_id'],
initiatorCustomerId: (int) $customer->id,
customPassword: !empty($payload['password']) ? (string) $payload['password'] : null,
initiator: 'client',
note: 'Client rebuild via service UUID: ' . $uuid,
);
} catch (Throwable $e) {
Log::error('Client rebuild dispatch failed', [
'service_uuid' => $uuid,
'module_vmware_id' => $vm->id,
'error' => $e->getMessage(),
]);
return response()->json([
'success' => false,
'message' => 'Không thể gửi yêu cầu rebuild: ' . $e->getMessage(),
], 400);
}
$this->activityLogger->log(
@@ -358,12 +360,21 @@ class ServiceController extends Controller
logName: 'VM',
description: "Customer {$customer->email} rebuild VM: {$vm->name} với OS ID: {$payload['os_id']} qua service UUID: {$uuid}",
subject: $vm,
newValues: ['os_id' => $payload['os_id'], 'service_uuid' => $uuid],
newValues: [
'os_id' => (int) $payload['os_id'],
'service_uuid' => $uuid,
'reinstall_log_id' => $log->id,
'custom_password' => !empty($payload['password']),
],
);
return response()->json([
'success' => true,
'message' => 'Đã gửi yêu cầu rebuild VM',
'data' => [
'reinstall_log_id' => $log->id,
'status' => $log->status,
],
]);
}
}
@@ -6,9 +6,12 @@ use App\Http\Controllers\Controller;
use App\Http\Controllers\Api\Service\Concerns\ResolvesVm;
use App\Http\Services\vCenter;
use App\Services\ActivityLogger;
use App\Services\VirtualMachine\Rebuild\VmRebuildService;
use Illuminate\Http\JsonResponse;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Throwable;
class VmRebuildController extends Controller
{
@@ -56,12 +59,13 @@ class VmRebuildController extends Controller
]);
}
public function rebuild(Request $request, vCenter $serviceVCenter): JsonResponse
public function rebuild(Request $request, VmRebuildService $rebuildService): JsonResponse
{
$customer = $request->user();
$payload = $request->validate([
'product' => ['required'],
'os_id' => ['required', 'integer', 'exists:os,id'],
'password' => ['nullable', 'string', 'min:8', 'max:255'],
]);
$vm = $this->resolveVmById($payload['product'], $request);
@@ -81,23 +85,25 @@ class VmRebuildController extends Controller
return response()->json(['error' => true, 'message' => 'OS không hợp lệ cho product hiện tại'], 422);
}
DB::table('reinstall_logs')->insert([
'product_id' => $vm->id,
'product_type' => 4,
'custome_id' => $customer->id,
'os_id' => $payload['os_id'],
'status' => 'pending',
'initiator' => 'client',
'note' => 'Client request rebuild from portal-client',
'created_at' => now(),
'updated_at' => now(),
]);
try {
$log = $rebuildService->dispatchRebuild(
moduleVmwareId: (int) $vm->id,
newOsId: (int) $payload['os_id'],
initiatorCustomerId: (int) $customer->id,
customPassword: !empty($payload['password']) ? (string) $payload['password'] : null,
initiator: 'client',
note: 'Client request rebuild from portal-client',
);
} catch (Throwable $e) {
Log::error('VmRebuildController: dispatchRebuild failed', [
'module_vmware_id' => $vm->id,
'new_os_id' => $payload['os_id'],
'error' => $e->getMessage(),
]);
$result = $serviceVCenter->reBuildVPS((string) $vm->name);
if (is_string($result) || ($result['error'] ?? false)) {
return response()->json([
'error' => true,
'message' => is_string($result) ? $result : ($result['message'] ?? 'Không thể rebuild VM'),
'error' => true,
'message' => 'Không thể gửi yêu cầu rebuild: ' . $e->getMessage(),
], 400);
}
@@ -112,7 +118,12 @@ class VmRebuildController extends Controller
return response()->json([
'error' => false,
'message' => 'Đã gửi yêu cầu rebuild VM',
'data' => $result,
'data' => [
'module_vmware_id' => $vm->id,
'new_os_id' => (int) $payload['os_id'],
'reinstall_log_id' => $log->id,
'status' => $log->status,
],
]);
}
Binary file not shown.
@@ -154,6 +154,25 @@ class AssignIpToGuestJob implements ShouldQueue
continue;
}
if (strtoupper($this->osFamily) === VmProvisionService::OS_WINDOWS) {
$this->persistIp($reservation, $moduleVmware, $ipPoolId, $serviceId);
Log::info('AssignIpToGuestJob: Windows IP assigned', [
'module_vmware_id' => $this->moduleVmwareId,
'vm_id' => $this->vmId,
'ip' => $ip,
'attempt' => $attempt,
]);
ChangePasswordViaVsphereJob::dispatch(
$this->moduleVmwareId,
$this->vmId,
$this->osFamily,
);
return;
}
// [6] Verify IP is active inside guest OS via PS1 script
$verifyOutput = $ssh->verifyIpInGuest(
vmId: $this->vmId,
@@ -0,0 +1,81 @@
<?php
declare(strict_types=1);
namespace App\Jobs\VirtualMachine\Rebuild;
use App\Services\VirtualMachine\Rebuild\VmRebuildAddonService;
use App\Services\VirtualMachine\Rebuild\VmRebuildService;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Throwable;
/**
* RebuildAddonInstallJob
*
* Reapplies CPU / RAM / Disk addons attached to the existing service onto
* the new VM. Idempotent — Upgrade* primitives are no-ops when the spec
* already matches.
*/
final class RebuildAddonInstallJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public int $tries = 3;
public int $timeout = 300;
public int $backoff = 30;
public function __construct(
public readonly int $moduleVmwareId,
public readonly int $newOsId,
public readonly string $newVmId,
public readonly int $reinstallLogId,
) {
}
public function handle(
VmRebuildService $rebuildService,
VmRebuildAddonService $addonService,
): void {
Log::info('RebuildAddonInstallJob: start', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
]);
$ctx = $rebuildService
->buildContext($this->moduleVmwareId, $this->newOsId, $this->newVmId);
$addonService->reinstall($ctx);
RebuildFinalizeJob::dispatch(
$this->moduleVmwareId,
$this->newOsId,
$this->newVmId,
$this->reinstallLogId,
)->onQueue('default');
Log::info('RebuildAddonInstallJob: completed, finalize dispatched', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
]);
}
public function failed(Throwable $e): void
{
Log::error('RebuildAddonInstallJob: failed — old VM preserved', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
'error' => $e->getMessage(),
]);
\App\Models\ReinstallLog::where('id', $this->reinstallLogId)->update([
'status' => \App\Models\ReinstallLog::STATUS_FAILED,
'error_message' => 'AddonInstall failed: ' . $e->getMessage(),
'updated_at' => now(),
]);
}
}
@@ -0,0 +1,92 @@
<?php
declare(strict_types=1);
namespace App\Jobs\VirtualMachine\Rebuild;
use App\Services\VirtualMachine\Rebuild\VmRebuildIpService;
use App\Services\VirtualMachine\Rebuild\VmRebuildService;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Throwable;
/**
* RebuildAssignIpJob
*
* Reuses the existing service IP on the new VM (no allocation, no pool
* mutation, no ownership change). Idempotent: re-running it simply runs the
* assign + verify scripts again with the same parameters.
*/
final class RebuildAssignIpJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public int $tries = 3;
public int $timeout = 600;
public int $backoff = 30;
public function __construct(
public readonly int $moduleVmwareId,
public readonly int $newOsId,
public readonly string $newVmId,
public readonly int $reinstallLogId,
) {
}
public function handle(
VmRebuildService $rebuildService,
VmRebuildIpService $ipService,
): void {
Log::info('RebuildAssignIpJob: start', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
'attempt' => $this->attempts(),
]);
$ctx = $rebuildService
->buildContext($this->moduleVmwareId, $this->newOsId, $this->newVmId);
try {
$ipService->assignAndVerify($ctx);
} catch (Throwable $e) {
// Surface the actual PS1 output into the reinstall log so admin
// can see WHY it failed without diving into laravel.log.
\App\Models\ReinstallLog::where('id', $this->reinstallLogId)->update([
'error_message' => 'AssignIp attempt ' . $this->attempts() . ': ' . $e->getMessage(),
'updated_at' => now(),
]);
throw $e;
}
RebuildChangePasswordJob::dispatch(
$this->moduleVmwareId,
$this->newOsId,
$this->newVmId,
$this->reinstallLogId,
)->onQueue('default');
Log::info('RebuildAssignIpJob: completed, password job dispatched', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
]);
}
public function failed(Throwable $e): void
{
Log::error('RebuildAssignIpJob: all retries exhausted — old VM preserved', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
'error' => $e->getMessage(),
]);
\App\Models\ReinstallLog::where('id', $this->reinstallLogId)->update([
'status' => \App\Models\ReinstallLog::STATUS_FAILED,
'error_message' => 'AssignIp failed: ' . $e->getMessage(),
'updated_at' => now(),
]);
}
}
@@ -0,0 +1,81 @@
<?php
declare(strict_types=1);
namespace App\Jobs\VirtualMachine\Rebuild;
use App\Services\VirtualMachine\Rebuild\VmRebuildPasswordService;
use App\Services\VirtualMachine\Rebuild\VmRebuildService;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Throwable;
/**
* RebuildChangePasswordJob
*
* Generates a fresh credential and applies it to the new VM via the shared
* VsphereHostSshService. Saves the encrypted + plaintext copy onto the
* existing ModuleVmware row.
*/
final class RebuildChangePasswordJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public int $tries = 5;
public int $timeout = 180;
public int $backoff = 60;
public function __construct(
public readonly int $moduleVmwareId,
public readonly int $newOsId,
public readonly string $newVmId,
public readonly int $reinstallLogId,
) {
}
public function handle(
VmRebuildService $rebuildService,
VmRebuildPasswordService $passwordService,
): void {
Log::info('RebuildChangePasswordJob: start', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
]);
$ctx = $rebuildService
->buildContext($this->moduleVmwareId, $this->newOsId, $this->newVmId);
$passwordService->rotate($ctx, $this->reinstallLogId);
RebuildAddonInstallJob::dispatch(
$this->moduleVmwareId,
$this->newOsId,
$this->newVmId,
$this->reinstallLogId,
)->onQueue('default');
Log::info('RebuildChangePasswordJob: completed, addon job dispatched', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
]);
}
public function failed(Throwable $e): void
{
Log::error('RebuildChangePasswordJob: failed — old VM preserved', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
'error' => $e->getMessage(),
]);
\App\Models\ReinstallLog::where('id', $this->reinstallLogId)->update([
'status' => \App\Models\ReinstallLog::STATUS_FAILED,
'error_message' => 'ChangePassword failed: ' . $e->getMessage(),
'updated_at' => now(),
]);
}
}
@@ -0,0 +1,103 @@
<?php
declare(strict_types=1);
namespace App\Jobs\VirtualMachine\Rebuild;
use App\Http\Services\vCenter as VCenterService;
use App\Models\ModuleVmware;
use App\Models\vCenter as VCenterModel;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Throwable;
/**
* RebuildDeleteOldVmJob
*
* Final step of the rebuild pipeline. Runs only AFTER the DB has already
* been swapped to point at the new VM, so even if this job fails the
* customer's service is fully online on the new VM — the old VM just
* becomes an orphan that an operator can clean up later.
*
* Defensive guard: before deleting, re-read ModuleVmware.vmware_id from DB
* and refuse to delete if it still matches the "old" id we were told about.
* That would mean the swap somehow rolled back, and deleting would destroy
* the live VM.
*/
final class RebuildDeleteOldVmJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public int $tries = 3;
public int $timeout = 180;
public int $backoff = 60;
public function __construct(
public readonly int $moduleVmwareId,
public readonly string $oldVmId,
public readonly ?int $reinstallLogId = null,
) {
}
public function handle(VCenterService $vCenter): void
{
if ($this->oldVmId === '') {
Log::info('RebuildDeleteOldVmJob: nothing to delete (oldVmId empty)');
return;
}
$vm = ModuleVmware::find($this->moduleVmwareId);
if (!$vm) {
Log::warning('RebuildDeleteOldVmJob: ModuleVmware vanished, skipping', [
'module_vmware_id' => $this->moduleVmwareId,
]);
return;
}
if ((string) $vm->vmware_id === $this->oldVmId) {
// DB swap never happened — refuse to delete the live VM.
Log::error('RebuildDeleteOldVmJob: ABORT — module_vmware still references oldVmId', [
'module_vmware_id' => $this->moduleVmwareId,
'old_vm_id' => $this->oldVmId,
]);
return;
}
$vcenter = VCenterModel::find($vm->vcenter_id);
if (!$vcenter || !$vCenter->loginServer($vcenter)) {
Log::error('RebuildDeleteOldVmJob: cannot login to vCenter', [
'old_vm_id' => $this->oldVmId,
]);
return;
}
try {
$vCenter->PowerOffVirtualMachine($this->oldVmId);
$res = $vCenter->DeleteVirtualMachine($this->oldVmId);
Log::info('RebuildDeleteOldVmJob: old VM deleted', [
'old_vm_id' => $this->oldVmId,
'result' => $res,
]);
} catch (Throwable $e) {
Log::error('RebuildDeleteOldVmJob: delete failed', [
'old_vm_id' => $this->oldVmId,
'error' => $e->getMessage(),
]);
// Re-throw so Laravel retries.
throw $e;
}
}
public function failed(Throwable $e): void
{
Log::error('RebuildDeleteOldVmJob: retries exhausted — manual cleanup required', [
'module_vmware_id' => $this->moduleVmwareId,
'old_vm_id' => $this->oldVmId,
'error' => $e->getMessage(),
]);
}
}
@@ -0,0 +1,184 @@
<?php
declare(strict_types=1);
namespace App\Jobs\VirtualMachine\Rebuild;
use App\Jobs\Mail\Admin\AdminOrderNotificationJob;
use App\Jobs\Mail\SendInformationVirtualMachine;
use App\Models\Invoice;
use App\Models\ModuleVmware;
use App\Models\Service;
use App\Models\ReinstallLog;
use App\Models\Setting;
use App\Services\VirtualMachine\Rebuild\VmRebuildService;
use App\Services\VirtualMachine\Rebuild\VmRebuildSwapService;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Throwable;
/**
* RebuildFinalizeJob
*
* Final validation + DB swap + credential email. Only after this step is the
* old VM safe to delete.
*
* [1] Health check the new VM in vCenter.
* [2] Swap module_vmware.vmware_id (+ virtual_machines.vm_id) atomically.
* [3] Send credentials email + admin notification.
* [4] Dispatch RebuildDeleteOldVmJob — deletion happens last.
*
* If health check fails: nothing is swapped, old VM stays attached to the
* service, and the operator gets a failed-job alert.
*/
final class RebuildFinalizeJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public int $tries = 3;
public int $timeout = 300;
public int $backoff = 30;
public function __construct(
public readonly int $moduleVmwareId,
public readonly int $newOsId,
public readonly string $newVmId,
public readonly int $reinstallLogId,
) {
}
public function handle(
VmRebuildService $rebuildService,
VmRebuildSwapService $swapService,
): void {
Log::info('RebuildFinalizeJob: start', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
]);
$ctx = $rebuildService
->buildContext($this->moduleVmwareId, $this->newOsId, $this->newVmId);
// [1] Health check before touching DB.
$swapService->healthCheck($ctx);
$oldVmId = $ctx->oldVmId;
// [2] Atomic DB swap.
$swapService->swapDatabaseReferences($ctx);
// [3] Notify customer with new credentials.
$this->sendCredentialsEmail($this->moduleVmwareId);
ReinstallLog::where('id', $this->reinstallLogId)->update([
'status' => ReinstallLog::STATUS_COMPLETED,
'new_vmware_id' => $ctx->newVmId,
'note' => 'Rebuild OK',
'completed_at' => now(),
'updated_at' => now(),
]);
// [4] Schedule old-VM deletion — last step, after everything is committed.
RebuildDeleteOldVmJob::dispatch(
$this->moduleVmwareId,
$oldVmId,
$this->reinstallLogId,
)->onQueue('default');
Log::info('RebuildFinalizeJob: completed', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
'old_vm_id' => $oldVmId,
]);
}
public function failed(Throwable $e): void
{
Log::error('RebuildFinalizeJob: failed — DB swap aborted, old VM remains attached', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $this->newVmId,
'error' => $e->getMessage(),
]);
ReinstallLog::where('id', $this->reinstallLogId)->update([
'status' => ReinstallLog::STATUS_FAILED,
'error_message' => $e->getMessage(),
'updated_at' => now(),
]);
}
private function sendCredentialsEmail(int $moduleVmwareId): void
{
$vm = ModuleVmware::with(['customer', 'os.osGroup', 'order.orderItems.service'])
->find($moduleVmwareId);
if (!$vm || !$vm->customer?->email) {
Log::warning('RebuildFinalizeJob: skip credentials email — VM/customer missing', [
'module_vmware_id' => $moduleVmwareId,
]);
return;
}
$setting = Setting::where('key', 'setting')->first();
$settingValue = $setting ? json_decode((string) $setting->value) : null;
$orderItem = $vm->order?->orderItems?->first();
$months = (int) ($orderItem?->months ?? 0);
$osGroup = $vm->os?->osGroup?->name ?? '';
$isWindows = str_contains(strtolower($osGroup), 'windows');
$osUsername = $isWindows ? 'Administrator' : 'root';
$emailData = [
'company_name' => $settingValue->company ?? 'Thế Giới Số',
'company_logo' => $settingValue->logo ?? '',
'client_name' => $vm->customer->fullname ?? '',
'product_name' => $orderItem?->getProductAttribute()?->name ?? 'Cloud VPS',
'product_billing' => $this->billingLabel($months),
'product_price' => $orderItem ? number_format((float) $orderItem->price, 0, ',', '.') : 'N/A',
'service_host' => $vm->ip ?? 'N/A',
'service_username' => $osUsername,
'service_port' => (string) ($vm->port ?? '22'),
'service_password' => $vm->plain_password ?? '(xem tai khoan quan ly)',
'os_name' => $vm->os ? ($osGroup . ' ' . $vm->os->version) : 'Unknown OS',
'service_due_exp' => $vm->next_due_date
? date('d/m/Y', (int) $vm->next_due_date)
: 'N/A',
'is_rebuild' => true,
];
$invoice = Invoice::where('order_id', $vm->order_id)->first();
SendInformationVirtualMachine::dispatch($emailData, $vm->customer, $invoice?->id);
AdminOrderNotificationJob::dispatch([
'customer_name' => $vm->customer->fullname ?? '',
'customer_email' => $vm->customer->email,
'client_ip' => null,
'order_num' => $vm->order?->order_num ?? $vm->order?->id ?? 'N/A',
'product_name' => ($orderItem?->getProductAttribute()?->name ?? 'Cloud VPS') . ' (Rebuild)',
'product_type' => 'Cloud VPS Rebuild',
'billing_cycle' => $this->billingLabel($months),
'total' => $orderItem?->price ?? 0,
'created_at' => now()->format('d/m/Y H:i:s'),
'ip' => $vm->ip ?? '',
'username' => $osUsername,
'password' => $vm->plain_password ?? '',
]);
}
private function billingLabel(int $months): string
{
return match ($months) {
1 => 'Hang thang',
3 => '3 Thang',
6 => '6 Thang',
12 => 'Hang nam',
24 => '2 Nam',
default => $months . ' Thang',
};
}
}
@@ -0,0 +1,129 @@
<?php
declare(strict_types=1);
namespace App\Jobs\VirtualMachine\Rebuild;
use App\Models\ModuleVmware;
use App\Models\ReinstallLog;
use App\Models\Service;
use App\Services\VirtualMachine\Rebuild\VmRebuildCloneService;
use App\Services\VirtualMachine\Rebuild\VmRebuildService;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Throwable;
/**
* RebuildVirtualMachineJob
*
* First step of the isolated rebuild pipeline.
*
* [1] Build RebuildContext from current DB state (idempotent reload).
* [2] Power off old VM (best-effort).
* [3] Clone new VM from chosen OS template.
* [4] Power on new VM + wait for VMware Tools.
* [5] Dispatch RebuildAssignIpJob with the new vSphere vm_id.
*
* On failure: old VM remains active, no DB references are swapped, no IP
* change, no deletion. ReinstallLog is marked failed; service is reverted
* to 'active' for visibility.
*/
final class RebuildVirtualMachineJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public int $tries = 1;
public int $timeout = 900;
public function __construct(
public readonly int $moduleVmwareId,
public readonly int $newOsId,
public readonly int $reinstallLogId,
) {
}
public function handle(
VmRebuildService $rebuildService,
VmRebuildCloneService $cloneService,
): void {
Log::info('RebuildVirtualMachineJob: start', [
'module_vmware_id' => $this->moduleVmwareId,
'new_os_id' => $this->newOsId,
'reinstall_log_id' => $this->reinstallLogId,
]);
try {
$ctx = $rebuildService->buildContext($this->moduleVmwareId, $this->newOsId);
$this->markServiceStatus($ctx->serviceId, 'rebuilding');
$cloneService->powerOffOldVm($ctx);
$newVmId = $cloneService->clone($ctx);
$ctx = $ctx->withNewVmId($newVmId);
$cloneService->powerOnAndWaitTools($ctx);
$this->markLog([
'status' => ReinstallLog::STATUS_IN_PROGRESS,
'new_vmware_id' => $newVmId,
'started_at' => now(),
]);
RebuildAssignIpJob::dispatch(
$this->moduleVmwareId,
$this->newOsId,
$newVmId,
$this->reinstallLogId,
)->onQueue('default');
Log::info('RebuildVirtualMachineJob: completed, IP job dispatched', [
'module_vmware_id' => $this->moduleVmwareId,
'new_vm_id' => $newVmId,
]);
} catch (Throwable $e) {
$this->onFailure($e);
throw $e;
}
}
public function failed(Throwable $e): void
{
$this->onFailure($e);
}
private function onFailure(Throwable $e): void
{
Log::error('RebuildVirtualMachineJob: failed — old VM preserved', [
'module_vmware_id' => $this->moduleVmwareId,
'error' => $e->getMessage(),
]);
ModuleVmware::where('id', $this->moduleVmwareId)->update(['status' => 1]);
$service = Service::where('vm_id', $this->moduleVmwareId)->first();
if ($service) {
$service->update(['status' => 'active']);
}
$this->markLog([
'status' => ReinstallLog::STATUS_FAILED,
'error_message' => $e->getMessage(),
]);
}
private function markServiceStatus(?int $serviceId, string $status): void
{
if ($serviceId) {
Service::where('id', $serviceId)->update(['status' => $status]);
}
}
private function markLog(array $attrs): void
{
ReinstallLog::where('id', $this->reinstallLogId)->update($attrs + ['updated_at' => now()]);
}
}
+52 -12
View File
@@ -1,34 +1,74 @@
<?php
declare(strict_types=1);
namespace App\Models;
use App\Http\Facade\SignRSA;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
/**
* ReinstallLog
*
* Audit row for every VM rebuild. Each row records:
* - which customer initiated it (custome_id — column kept for legacy compat)
* - which service / VM was rebuilt
* - which OS was chosen
* - the new credential applied to the rebuilt VM (encrypted + plaintext)
* - status lifecycle: pending → in_progress → completed | failed
*/
class ReinstallLog extends Model
{
use HasFactory;
protected $fillable = [
'product_id',
'product_type',
'customer_id',
'os_id',
'template_id',
'ip_address',
'status',
'error_message',
'initiator',
'note',
protected $table = 'reinstall_logs';
protected $guarded = [];
protected $casts = [
'started_at' => 'datetime',
'completed_at' => 'datetime',
'password_supplied_by_customer' => 'boolean',
];
public const PRODUCT_TYPE_VPS = 4;
public const STATUS_PENDING = 'pending';
public const STATUS_IN_PROGRESS = 'in_progress';
public const STATUS_COMPLETED = 'completed';
public const STATUS_FAILED = 'failed';
public function customer()
{
return $this->belongsTo(Customer::class);
return $this->belongsTo(Customer::class, 'custome_id');
}
public function service()
{
return $this->belongsTo(Service::class, 'service_id');
}
public function vm()
{
return $this->belongsTo(ModuleVmware::class, 'product_id');
}
public function os()
{
return $this->belongsTo(Os::class, 'os_id');
}
/** Decrypt the password column on demand (null if not yet set). */
public function decryptedPassword(): ?string
{
if (!$this->password) {
return $this->plain_password ?: null;
}
try {
return SignRSA::DecryptFunc($this->password);
} catch (\Throwable $e) {
return $this->plain_password ?: null;
}
}
}
@@ -0,0 +1,90 @@
<?php
declare(strict_types=1);
namespace App\Services\VirtualMachine\Rebuild;
use App\DTOs\VirtualMachine\Rebuild\RebuildContext;
use App\Http\Services\vCenter as VCenterService;
use App\Models\ModuleVmware;
use App\Models\vCenter as VCenterModel;
use Illuminate\Support\Facades\Log;
use RuntimeException;
/**
* VmRebuildAddonService
*
* Reapplies CPU / RAM / Disk addons attached to the existing service onto
* the freshly cloned VM. Uses vCenter's existing Upgrade* primitives but is
* fully independent from VmAddonResolver / CreateVirtualMachineJob.
*/
final class VmRebuildAddonService
{
public function __construct(private readonly VCenterService $vCenter)
{
}
public function reinstall(RebuildContext $ctx): void
{
if ($ctx->newVmId === null) {
throw new RuntimeException('Rebuild addons: newVmId is null');
}
$vcenter = VCenterModel::find($ctx->vcenterId);
if (!$vcenter || !$this->vCenter->loginServer($vcenter)) {
throw new RuntimeException(
"Rebuild addons: cannot login to vCenter #{$ctx->vcenterId}"
);
}
// Resolve the totals we want the rebuilt VM to end up with.
// Base spec ensures CPU/RAM/Disk parity even if there are no addons.
$targetCpu = max($ctx->baseCpu, (int) ($ctx->addonTotals['cpu'] ?? 0));
$targetRam = max($ctx->baseRam, (int) ($ctx->addonTotals['ram'] ?? 0));
$targetDisk = max($ctx->baseDisk, (int) ($ctx->addonTotals['disk'] ?? 0));
Log::info('VmRebuildAddonService: applying specs', [
'module_vmware_id' => $ctx->moduleVmwareId,
'new_vm_id' => $ctx->newVmId,
'cpu' => $targetCpu,
'ram' => $targetRam,
'disk' => $targetDisk,
]);
if ($targetCpu > $ctx->baseCpu) {
$res = $this->vCenter->UpgradeCPUVirtualMachine($ctx->newVmId, $targetCpu);
$this->ensureOk($res, 'UpgradeCPU');
}
if ($targetRam > $ctx->baseRam) {
$res = $this->vCenter->UpgradeMemoryVirtualMachine($ctx->newVmId, $targetRam);
$this->ensureOk($res, 'UpgradeMemory');
}
if ($targetDisk > $ctx->baseDisk) {
// UpgradeDiskVirtualMachine expects a "product" with vmware_id —
// build a transient ModuleVmware-like object pointing at the new VM.
$shim = new ModuleVmware();
$shim->vmware_id = $ctx->newVmId;
$res = $this->vCenter->UpgradeDiskVirtualMachine($shim, $targetDisk);
$this->ensureOk($res, 'UpgradeDisk');
}
Log::info('VmRebuildAddonService: addons reinstalled', [
'module_vmware_id' => $ctx->moduleVmwareId,
'new_vm_id' => $ctx->newVmId,
]);
}
private function ensureOk(array|string $result, string $op): void
{
if (is_string($result)) {
throw new RuntimeException("Rebuild addons: {$op} returned '{$result}'");
}
if (($result['error'] ?? false)) {
throw new RuntimeException(
"Rebuild addons: {$op} failed — " . json_encode($result),
);
}
}
}
@@ -0,0 +1,204 @@
<?php
declare(strict_types=1);
namespace App\Services\VirtualMachine\Rebuild;
use App\DTOs\VirtualMachine\Rebuild\RebuildContext;
use App\Http\Services\vCenter as VCenterService;
use App\Models\ModuleDatastore;
use App\Models\vCenter as VCenterModel;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Log;
use RuntimeException;
/**
* VmRebuildCloneService
*
* Clones a fresh VM from the chosen OS template and powers it on.
* Uses the vCenter REST API directly — never dispatches CreateVirtualMachineJob.
*/
final class VmRebuildCloneService
{
private const CLONE_TIMEOUT = 300;
private const POWER_ON_TIMEOUT = 30;
private const TOOLS_POLL_DELAY = 10;
private const TOOLS_MAX_ATTEMPTS = 30;
/**
* Extra grace after VMware Tools reports an IP. VMware Tools advertises
* the guest IP as soon as the kernel finishes networking, but
* Invoke-VMScript / guest ops daemon needs another ~20–40 s before it
* actually accepts script invocations on a fresh clone. Without this
* delay assignIP/verifyIP scripts often return "FAIL" on the first run.
*/
private const TOOLS_READY_GRACE_SECONDS = 45;
public function __construct(private readonly VCenterService $vCenter)
{
}
/**
* Clone a new VM from the OS template specified in $ctx.
*
* @return string vSphere vm_id of the newly cloned VM
*/
public function clone(RebuildContext $ctx): string
{
$vcenter = VCenterModel::find($ctx->vcenterId);
if (!$vcenter || !$this->vCenter->loginServer($vcenter)) {
throw new RuntimeException("Rebuild clone: cannot login to vCenter #{$ctx->vcenterId}");
}
$datastore = ModuleDatastore::where('module_host_id', $ctx->hostId)->first();
$datastoreRef = $datastore?->datastore ?: 'datastore-3004';
$body = [
'spec' => [
'name' => $ctx->transientName,
'source' => $ctx->template,
'placement' => [
'datacenter' => $ctx->datacenterRef,
'folder' => 'group-v1008',
'host' => $ctx->hostRef,
'resource_pool' => 'resgroup-3002',
'datastore' => $datastoreRef,
],
'power_on' => false,
'fallback_to_datastore_default_policy' => true,
],
];
Log::info('VmRebuildCloneService: clone start', [
'module_vmware_id' => $ctx->moduleVmwareId,
'transient_name' => $ctx->transientName,
'template' => $ctx->template,
]);
$response = Http::withoutVerifying()
->timeout(self::CLONE_TIMEOUT)
->connectTimeout(30)
->withHeaders([
'vmware-api-session-id' => $this->vCenter->token,
'Content-Type' => 'application/json',
])
->post(
"https://{$this->vCenter->apiUri}/rest/vcenter/vm?action=clone",
$body,
);
$data = $response->json();
if (!$response->successful() || isset($data['type'])) {
throw new RuntimeException(
'Rebuild clone failed: ' . json_encode($data ?: $response->body())
);
}
$newVmId = (string) ($data['value'] ?? '');
if ($newVmId === '' || in_array($newVmId, ['unknown', 'UNKNOWN'], true)) {
throw new RuntimeException('Rebuild clone: empty vm_id returned by vCenter');
}
Log::info('VmRebuildCloneService: clone success', [
'module_vmware_id' => $ctx->moduleVmwareId,
'new_vm_id' => $newVmId,
]);
return $newVmId;
}
/**
* Power on the freshly cloned VM and wait for VMware Tools to report
* a guest IP — without this, IP assignment scripts cannot run.
*/
public function powerOnAndWaitTools(RebuildContext $ctx): void
{
if ($ctx->newVmId === null) {
throw new RuntimeException('powerOnAndWaitTools: newVmId is null');
}
$vcenter = VCenterModel::find($ctx->vcenterId);
if (!$vcenter || !$this->vCenter->loginServer($vcenter)) {
throw new RuntimeException("Rebuild powerOn: cannot login to vCenter #{$ctx->vcenterId}");
}
$powerResp = Http::withoutVerifying()
->timeout(self::POWER_ON_TIMEOUT)
->withHeaders(['vmware-api-session-id' => $this->vCenter->token])
->post("https://{$this->vCenter->apiUri}/rest/vcenter/vm/{$ctx->newVmId}/power/start");
if (!$powerResp->successful() && $powerResp->status() !== 400) {
// 400 = already powered on; treat as OK.
throw new RuntimeException(
"Rebuild powerOn failed: status={$powerResp->status()} body={$powerResp->body()}"
);
}
Log::info('VmRebuildCloneService: powered on, waiting for tools', [
'new_vm_id' => $ctx->newVmId,
]);
for ($i = 1; $i <= self::TOOLS_MAX_ATTEMPTS; $i++) {
$info = Http::withoutVerifying()
->timeout(10)
->withHeaders(['vmware-api-session-id' => $this->vCenter->token])
->get("https://{$this->vCenter->apiUri}/rest/vcenter/vm/{$ctx->newVmId}/guest/identity")
->json();
$guestIp = trim((string) ($info['value']['ip_address'] ?? ''));
if ($guestIp !== '' && $guestIp !== '0.0.0.0') {
Log::info('VmRebuildCloneService: tools ready — applying grace', [
'new_vm_id' => $ctx->newVmId,
'guest_ip' => $guestIp,
'attempt' => $i,
'grace_seconds' => self::TOOLS_READY_GRACE_SECONDS,
]);
// Give the guest ops daemon time to accept Invoke-VMScript.
sleep(self::TOOLS_READY_GRACE_SECONDS);
return;
}
sleep(self::TOOLS_POLL_DELAY);
}
throw new RuntimeException(
"Rebuild powerOn: VMware Tools never became ready on new VM {$ctx->newVmId}"
);
}
/**
* Best-effort: power off the OLD VM so it stops responding on the shared IP
* before we reassign it inside the guest of the new VM. We do NOT delete it
* here — deletion is the very last step, after full validation.
*/
public function powerOffOldVm(RebuildContext $ctx): void
{
if ($ctx->oldVmId === '') {
return;
}
$vcenter = VCenterModel::find($ctx->vcenterId);
if (!$vcenter || !$this->vCenter->loginServer($vcenter)) {
Log::warning('VmRebuildCloneService: powerOff old VM skipped — vCenter login failed', [
'old_vm_id' => $ctx->oldVmId,
]);
return;
}
try {
Http::withoutVerifying()
->timeout(30)
->withHeaders(['vmware-api-session-id' => $this->vCenter->token])
->post("https://{$this->vCenter->apiUri}/rest/vcenter/vm/{$ctx->oldVmId}/power/stop");
Log::info('VmRebuildCloneService: old VM powered off', [
'old_vm_id' => $ctx->oldVmId,
]);
} catch (\Throwable $e) {
// Non-fatal — guest may already be off, or vCenter may flake.
Log::warning('VmRebuildCloneService: powerOff old VM failed', [
'old_vm_id' => $ctx->oldVmId,
'error' => $e->getMessage(),
]);
}
}
}
@@ -0,0 +1,219 @@
<?php
declare(strict_types=1);
namespace App\Services\VirtualMachine\Rebuild;
use App\DTOs\VirtualMachine\Rebuild\RebuildContext;
use App\Http\Facade\SignRSA;
use App\Models\IpPool;
use App\Models\ModuleVmware;
use App\Models\Os;
use App\Models\vCenter as VCenterModel;
use App\Services\VsphereHostSshService;
use Illuminate\Support\Facades\Log;
use RuntimeException;
/**
* VmRebuildIpService
*
* Mirrors the production AssignIpToGuestJob flow 1:1 for the rebuild path —
* uses the SAME VsphereHostSshService helpers, which means the SAME PS1
* scripts on the vSphere host:
*
* C:\CLISave\AssignIP\assignIP{Linux|Windows}.ps1
* C:\CLISave\AssignIP\verifyIP{Linux|Windows}.ps1
*
* Differences vs create flow (hard rules, per the rebuild spec):
* - NEVER reserve a new IP
* - NEVER mutate ip_pools / module_ip_address ownership
* - The IP, gateway, cidr, dns are read from the EXISTING service:
* gateway/cidr ← IpPool row whose ip_address == current VM IP
* dns ← product.ipAddress.dns (same as AssignIpToGuestJob)
* hostId ← osGroup.host.id of the NEW OS (same as AssignIpToGuestJob)
*
* In-place retry loop because we MUST reuse the same IP — we can't fall
* through to a different IP like AssignIpToGuestJob does.
*/
final class VmRebuildIpService
{
private const MAX_ATTEMPTS = 4;
private const ATTEMPT_DELAY = 25;
public function __construct(private readonly VsphereHostSshService $ssh)
{
}
public function assignAndVerify(RebuildContext $ctx): void
{
if ($ctx->newVmId === null) {
throw new RuntimeException('Rebuild IP: newVmId is null');
}
if ($ctx->ip === '') {
throw new RuntimeException('Rebuild IP: missing existing IP in context');
}
// Load ModuleVmware + relations EXACTLY like AssignIpToGuestJob does
// so we read parameters from the same DB locations.
$moduleVmware = ModuleVmware::with([
'order.orderItems.product.ipAddress',
'vcenter',
])->find($ctx->moduleVmwareId);
if (!$moduleVmware) {
throw new RuntimeException(
"Rebuild IP: ModuleVmware #{$ctx->moduleVmwareId} not found"
);
}
// DNS comes from product.ipAddress.dns — same source AssignIpToGuestJob uses.
$orderItem = $moduleVmware->order?->orderItems?->first();
$dns = $orderItem?->product?->ipAddress?->dns
?: ($ctx->dns ?: '8.8.8.8');
// Host id is osGroup.host.id of the NEW OS — same source AssignIpToGuestJob uses
// (it loads Os->osGroup->host for whatever OS is on the order item).
$osModel = Os::with('osGroup.host')->find($ctx->newOsId);
$hostId = $osModel?->osGroup?->host?->id ?? $ctx->hostId ?? 1;
// Gateway + CIDR come from the ip_pools row already assigned to this
// service — same DB source the create flow reads at reservation time.
// For rebuild we never reserve; we just look it up by ip_address.
$ipPool = IpPool::where('ip_address', $ctx->ip)->first();
$gateway = (string) ($ipPool?->gateway ?? $ctx->gateway);
$cidr = (string) ((int) ($ipPool?->cidr ?? $ctx->cidr ?: 24));
if ($gateway === '') {
throw new RuntimeException(
"Rebuild IP: cannot resolve gateway for IP {$ctx->ip} (no ip_pools row and context empty)"
);
}
// vCenter credentials — identical decode path as AssignIpToGuestJob.
$vcenter = $moduleVmware->vcenter
?? VCenterModel::find($ctx->vcenterId)
?? VCenterModel::first();
if (!$vcenter) {
throw new RuntimeException('Rebuild IP: no vCenter record available');
}
$vcenterHost = (string) $vcenter->host;
$vcenterUser = (string) $vcenter->user;
$vcenterPass = SignRSA::DecryptFunc($vcenter->password);
// Guest OS default admin user per OS family — same convention.
$guestUser = strtoupper($ctx->osFamily) === 'WINDOWS' ? 'Administrator' : 'root';
Log::info('VmRebuildIpService: assigning existing IP to new VM', [
'module_vmware_id' => $ctx->moduleVmwareId,
'new_vm_id' => $ctx->newVmId,
'ip' => $ctx->ip,
'gateway' => $gateway,
'cidr' => $cidr,
'dns' => $dns,
'host_id' => $hostId,
'os_family' => $ctx->osFamily,
'guest_user' => $guestUser,
]);
$lastAssignOutput = '';
$lastVerifyOutput = '';
for ($attempt = 1; $attempt <= self::MAX_ATTEMPTS; $attempt++) {
// [4] Call assignIP{Linux|Windows}.ps1 — IDENTICAL invocation
// to AssignIpToGuestJob, same VsphereHostSshService::assignIp().
$lastAssignOutput = $this->ssh->assignIp(
vmId: $ctx->newVmId,
ip: $ctx->ip,
gateway: $gateway,
cidr: $cidr,
vcenterHost: $vcenterHost,
vcenterUser: $vcenterUser,
vcenterPassword: $vcenterPass,
guestPassword: VmRebuildService::TEMPLATE_PASSWORD,
osFamily: $ctx->osFamily,
dns: $dns,
guestUser: $guestUser,
);
if (!$this->isSuccess($lastAssignOutput)) {
Log::warning('VmRebuildIpService: assignIp FAIL', [
'ip' => $ctx->ip,
'attempt' => $attempt,
'new_vm_id' => $ctx->newVmId,
'output' => trim($lastAssignOutput),
]);
if ($attempt < self::MAX_ATTEMPTS) {
sleep(self::ATTEMPT_DELAY);
continue;
}
throw new RuntimeException(
'Rebuild IP: assign script failed after ' . self::MAX_ATTEMPTS
. ' attempts — last output: ' . trim($lastAssignOutput),
);
}
if (strtoupper($ctx->osFamily) === VmRebuildService::OS_WINDOWS) {
Log::info('VmRebuildIpService: Windows IP assigned', [
'module_vmware_id' => $ctx->moduleVmwareId,
'new_vm_id' => $ctx->newVmId,
'ip' => $ctx->ip,
'attempt' => $attempt,
]);
return;
}
// [6] Call verifyIP{Linux|Windows}.ps1 — same as AssignIpToGuestJob.
$lastVerifyOutput = $this->ssh->verifyIpInGuest(
vmId: $ctx->newVmId,
ip: $ctx->ip,
vcenterHost: $vcenterHost,
vcenterUser: $vcenterUser,
vcenterPassword: $vcenterPass,
guestPassword: VmRebuildService::TEMPLATE_PASSWORD,
osFamily: $ctx->osFamily,
guestUser: $guestUser,
);
if ($this->isSuccess($lastVerifyOutput)) {
Log::info('VmRebuildIpService: IP assigned and verified', [
'module_vmware_id' => $ctx->moduleVmwareId,
'new_vm_id' => $ctx->newVmId,
'ip' => $ctx->ip,
'attempt' => $attempt,
]);
return;
}
Log::warning('VmRebuildIpService: verifyIpInGuest FAIL', [
'ip' => $ctx->ip,
'attempt' => $attempt,
'new_vm_id' => $ctx->newVmId,
'output' => trim($lastVerifyOutput),
]);
if ($attempt < self::MAX_ATTEMPTS) {
sleep(self::ATTEMPT_DELAY);
}
}
throw new RuntimeException(
'Rebuild IP: verify script failed after ' . self::MAX_ATTEMPTS
. ' attempts — last output: ' . trim($lastVerifyOutput),
);
}
/** PS1 scripts return JSON {"status":"ok"} or plain SUCCESS — same parser as AssignIpToGuestJob. */
private function isSuccess(string $output): bool
{
if (str_contains(strtoupper(trim($output)), 'SUCCESS')) {
return true;
}
$data = json_decode(trim($output), true);
return is_array($data) && ($data['status'] ?? '') === 'ok';
}
}
@@ -0,0 +1,126 @@
<?php
declare(strict_types=1);
namespace App\Services\VirtualMachine\Rebuild;
use App\DTOs\VirtualMachine\Rebuild\RebuildContext;
use App\Http\Facade\SignRSA;
use App\Models\ModuleVmware;
use App\Models\ReinstallLog;
use App\Models\vCenter as VCenterModel;
use App\Services\VsphereHostSshService;
use Illuminate\Support\Facades\Log;
use RuntimeException;
/**
* VmRebuildPasswordService
*
* Generates a new random password for the rebuilt VM and pushes it through
* VsphereHostSshService::changePassword(). Uses the same low-level SSH helper
* as the create flow but never dispatches ChangePasswordViaVsphereJob.
*
* The new credential is persisted on the EXISTING ModuleVmware row — the
* service identity is preserved across rebuild.
*/
final class VmRebuildPasswordService
{
public function __construct(private readonly VsphereHostSshService $ssh)
{
}
/**
* @param int|null $reinstallLogId If supplied, the customer-supplied
* password (if any) is pulled from this
* log row and the same row is updated
* with the credential that was applied.
* @return string the plaintext password actually applied
*/
public function rotate(RebuildContext $ctx, ?int $reinstallLogId = null): string
{
if ($ctx->newVmId === null) {
throw new RuntimeException('Rebuild password: newVmId is null');
}
$vcenter = VCenterModel::find($ctx->vcenterId);
if (!$vcenter) {
throw new RuntimeException("Rebuild password: vCenter #{$ctx->vcenterId} not found");
}
// Prefer a customer-supplied credential when one was provided.
$newPassword = null;
$log = $reinstallLogId ? ReinstallLog::find($reinstallLogId) : null;
if ($log && $log->password_supplied_by_customer && $log->password) {
try {
$newPassword = SignRSA::DecryptFunc($log->password);
} catch (\Throwable $e) {
$newPassword = null;
}
}
if (!is_string($newPassword) || $newPassword === '') {
$newPassword = $this->generatePassword(16);
}
$output = $this->ssh->changePassword(
$ctx->newVmId,
$newPassword,
$ctx->osFamily,
(string) $vcenter->host,
(string) $vcenter->user,
SignRSA::DecryptFunc($vcenter->password),
);
if (!str_contains(strtoupper(trim($output)), 'SUCCESS')) {
throw new RuntimeException(
"Rebuild password: changePassword script failed — " . trim($output),
);
}
$encrypted = SignRSA::EncryptFunc($newPassword);
ModuleVmware::where('id', $ctx->moduleVmwareId)->update([
'password' => $encrypted,
'plain_password' => $newPassword,
]);
if ($log) {
$log->update([
'password' => $encrypted,
'plain_password' => $newPassword,
]);
}
Log::info('VmRebuildPasswordService: password rotated', [
'module_vmware_id' => $ctx->moduleVmwareId,
'new_vm_id' => $ctx->newVmId,
'reinstall_log_id' => $log?->id,
]);
return $newPassword;
}
private function generatePassword(int $length): string
{
$sets = [
'ABCDEFGHJKLMNPQRSTUVWXYZ',
'abcdefghjkmnpqrstuvwxyz',
'23456789',
'!@#$*',
];
$password = '';
$allChars = implode('', $sets);
foreach ($sets as $set) {
$password .= $set[random_int(0, strlen($set) - 1)];
}
for ($i = count($sets); $i < $length; $i++) {
$password .= $allChars[random_int(0, strlen($allChars) - 1)];
}
$chars = str_split($password);
for ($i = count($chars) - 1; $i > 0; $i--) {
$j = random_int(0, $i);
[$chars[$i], $chars[$j]] = [$chars[$j], $chars[$i]];
}
return implode('', $chars);
}
}
@@ -0,0 +1,276 @@
<?php
declare(strict_types=1);
namespace App\Services\VirtualMachine\Rebuild;
use App\DTOs\VirtualMachine\Rebuild\RebuildContext;
use App\Http\Facade\SignRSA;
use App\Jobs\VirtualMachine\Rebuild\RebuildVirtualMachineJob;
use App\Models\ModuleHost;
use App\Models\ModuleVmware;
use App\Models\Os;
use App\Models\ReinstallLog;
use App\Models\Service;
use App\Models\ServiceAddon;
use App\Models\vCenter;
use Illuminate\Support\Facades\Log;
use RuntimeException;
/**
* VmRebuildService
*
* Orchestrator/entry point for the rebuild subsystem. Builds the
* RebuildContext from the existing VM + chosen new OS and dispatches
* the first rebuild job.
*
* Completely isolated from VmProvisionService / VmInfrastructureResolver
* and from the CreateVirtualMachineJob chain.
*/
final class VmRebuildService
{
public const OS_LINUX = 'LINUX';
public const OS_WINDOWS = 'WINDOWS';
/**
* Default password baked into rebuild templates. Mirrors the create-flow
* template password constant but is duplicated intentionally to keep this
* subsystem standalone.
*/
public const TEMPLATE_PASSWORD = 'TGS@123';
/**
* Build a fresh RebuildContext from current DB state.
*
* @param int|null $newVmIdOverride Pass when the new VM has already been cloned
* and we need context for a downstream job.
*/
public function buildContext(
int $moduleVmwareId,
int $newOsId,
?string $newVmIdOverride = null,
): RebuildContext {
$vm = ModuleVmware::with([
'vCenter',
'ipAddress',
'order.orderItems.product',
])->find($moduleVmwareId);
if (!$vm) {
throw new RuntimeException("Rebuild: ModuleVmware #{$moduleVmwareId} not found");
}
$service = Service::where('vm_id', $vm->id)->first();
$os = Os::with(['host.datacenter.vCenter', 'osGroup.host.datacenter.vCenter'])
->find($newOsId);
if (!$os) {
throw new RuntimeException("Rebuild: target OS #{$newOsId} not found");
}
$orderItem = $vm->order?->orderItems?->first();
$product = $orderItem?->product;
// Resolve host / datacenter / vcenter for the chosen OS.
$host = $os->host ?? $os->osGroup?->host;
$datacenter = $host?->datacenter;
$vcenter = $datacenter?->vCenter;
if ((!$host || !$datacenter || !$vcenter) && $product && $product->module_host_id) {
$productHost = ModuleHost::with('datacenter.vCenter')->find($product->module_host_id);
$host = $host ?? $productHost;
$datacenter = $datacenter ?? $productHost?->datacenter;
$vcenter = $vcenter ?? $productHost?->datacenter?->vCenter;
}
// Final fallback: original VM's vCenter (IP must stay on the same fabric).
$vcenter = $vcenter ?? $vm->vCenter ?? vCenter::find($vm->vcenter_id);
if (!$vcenter || !$host || !$datacenter) {
throw new RuntimeException(
"Rebuild: cannot resolve host/datacenter/vcenter for OS #{$newOsId}"
);
}
$osGroupName = $os->osGroup?->name ?? '';
$isWindows = str_contains(strtolower($osGroupName), 'windows');
// Base specs come from the original product so the VM keeps its
// CPU/RAM/disk after rebuild.
$baseCpu = max(1, (int) ($product->cpu ?? 1));
$baseRam = max(1, (int) ($product->ram ?? 1));
$baseDisk = max(20, (int) ($product->disk ?? 20));
// Existing IP must be preserved verbatim — pull from module_vmware
// first, then fall back to the linked ModuleIpAddress record.
$ip = (string) ($vm->ip ?? $vm->ipAddress?->ip ?? '');
$gateway = (string) ($vm->ipAddress?->gateway ?? '');
$netmask = (string) ($vm->ipAddress?->netmask ?? '');
$dns = (string) ($vm->dns ?? $vm->ipAddress?->dns ?? '8.8.8.8');
$cidr = $netmask !== '' ? $this->netmaskToCidr($netmask) : 24;
if ($ip === '') {
throw new RuntimeException(
"Rebuild: ModuleVmware #{$moduleVmwareId} has no IP — cannot reuse"
);
}
// Addon totals for reinstall step (CPU/RAM/Disk delta on top of base).
$addonTotals = $this->collectAddonTotals($service);
return new RebuildContext(
moduleVmwareId: $vm->id,
serviceId: $service?->id,
customerId: (int) $vm->customer_id,
newOsId: (int) $os->id,
oldVmId: (string) $vm->vmware_id,
newVmId: $newVmIdOverride,
vmName: (string) ($vm->name ?? ''),
transientName: $this->makeTransientName((string) ($vm->name ?? ''), $vm->id),
ip: $ip,
gateway: $gateway,
netmask: $netmask,
cidr: $cidr,
dns: $dns,
moduleIpAddressId: $vm->module_ip_address ? (int) $vm->module_ip_address : null,
vcenterId: (int) $vcenter->id,
hostId: (int) $host->id,
isWindows: $isWindows,
osFamily: $isWindows ? self::OS_WINDOWS : self::OS_LINUX,
guestUser: $isWindows ? 'Administrator' : 'root',
template: $os->template_id,
datacenterRef: $datacenter->datacenter_id ?? null,
hostRef: $host->host_id ?? null,
baseCpu: $baseCpu,
baseRam: $baseRam,
baseDisk: $baseDisk,
addonTotals: $addonTotals,
);
}
/**
* Entry point used by the controller — kicks off the queue chain and
* returns the ReinstallLog row that tracks this rebuild from end to end.
*
* @param string|null $customPassword Optional customer-supplied credential.
* When set, the rebuild applies this
* exact password instead of generating
* a random one.
* @param string $initiator 'client' | 'admin' | 'system'
* @param string|null $note Free-form note to record on the log row.
*/
public function dispatchRebuild(
int $moduleVmwareId,
int $newOsId,
int $initiatorCustomerId,
?string $customPassword = null,
string $initiator = 'client',
?string $note = null,
): ReinstallLog {
// Build context once for validation, then let the job rebuild it on its own.
$ctx = $this->buildContext($moduleVmwareId, $newOsId);
if ((int) $ctx->customerId !== $initiatorCustomerId) {
throw new RuntimeException('Rebuild: customer mismatch — refusing to dispatch.');
}
$log = $this->openLog($ctx, $initiatorCustomerId, $customPassword, $initiator, $note);
Log::info('VmRebuildService: dispatching rebuild', [
'module_vmware_id' => $ctx->moduleVmwareId,
'service_id' => $ctx->serviceId,
'old_vm_id' => $ctx->oldVmId,
'new_os_id' => $ctx->newOsId,
'ip' => $ctx->ip,
'reinstall_log_id' => $log->id,
]);
RebuildVirtualMachineJob::dispatch($ctx->moduleVmwareId, $ctx->newOsId, $log->id)
->onQueue('default');
return $log;
}
/**
* Create the ReinstallLog row that the rest of the pipeline updates.
* Any older 'pending' rows for the same VM (created by legacy paths) are
* marked as superseded so admin views show a single active record.
*/
private function openLog(
RebuildContext $ctx,
int $customerId,
?string $customPassword,
string $initiator,
?string $note,
): ReinstallLog {
ReinstallLog::query()
->where('product_id', $ctx->moduleVmwareId)
->where('product_type', ReinstallLog::PRODUCT_TYPE_VPS)
->whereIn('status', [ReinstallLog::STATUS_PENDING, ReinstallLog::STATUS_IN_PROGRESS])
->update([
'status' => ReinstallLog::STATUS_FAILED,
'note' => 'Superseded by newer rebuild request',
'updated_at' => now(),
]);
return ReinstallLog::create([
'product_id' => $ctx->moduleVmwareId,
'product_type' => ReinstallLog::PRODUCT_TYPE_VPS,
'custome_id' => $customerId,
'service_id' => $ctx->serviceId,
'os_id' => $ctx->newOsId,
'template_id' => is_string($ctx->template) ? $ctx->template : null,
'ip_address' => $ctx->ip,
'old_vmware_id' => $ctx->oldVmId,
'status' => ReinstallLog::STATUS_PENDING,
'initiator' => $initiator,
'note' => $note,
'password' => $customPassword ? SignRSA::EncryptFunc($customPassword) : null,
'plain_password' => null, // filled when password is actually applied
'password_supplied_by_customer' => $customPassword !== null,
]);
}
/**
* @return array{cpu:int,ram:int,disk:int,bandwidth:int}
*/
private function collectAddonTotals(?Service $service): array
{
$totals = ['cpu' => 0, 'ram' => 0, 'disk' => 0, 'bandwidth' => 0];
if (!$service) {
return $totals;
}
$addons = ServiceAddon::where('service_id', $service->id)->get();
foreach ($addons as $addon) {
$type = strtolower((string) $addon->addon_type);
$val = (int) $addon->addon_value;
if (array_key_exists($type, $totals)) {
// Base specs were inserted as ServiceAddon rows by the create
// flow with price=0 and value=base. Treat the *first* row per
// type as base (overwrite), and any extras as additive.
$totals[$type] = $totals[$type] === 0 ? $val : $totals[$type] + $val;
}
}
return $totals;
}
private function makeTransientName(string $vmName, int $moduleVmwareId): string
{
$base = $vmName !== '' ? $vmName : ('VM-' . $moduleVmwareId);
$suffix = '-rb-' . substr((string) time(), -6);
// Keep total length sane for vCenter (<= 80)
return substr($base, 0, 80 - strlen($suffix)) . $suffix;
}
private function netmaskToCidr(string $netmask): int
{
$bits = 0;
foreach (explode('.', $netmask) as $octet) {
$bits += substr_count(decbin((int) $octet), '1');
}
return $bits > 0 ? $bits : 24;
}
}
@@ -0,0 +1,160 @@
<?php
declare(strict_types=1);
namespace App\Services\VirtualMachine\Rebuild;
use App\DTOs\VirtualMachine\Rebuild\RebuildContext;
use App\Http\Services\vCenter as VCenterService;
use App\Models\ModuleVmware;
use App\Models\Service;
use App\Models\vCenter as VCenterModel;
use App\Models\VirtualMachines;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Throwable;
/**
* VmRebuildSwapService
*
* After the new VM has passed every validation step, this service:
* 1. Performs a final health check against the new VM in vCenter.
* 2. Atomically swaps DB references from the old vmware_id to the new one.
* 3. Deletes the old VM from vCenter — and ONLY then.
*
* If health check fails, nothing is swapped and the old VM is preserved
* untouched so a manual recovery is possible.
*/
final class VmRebuildSwapService
{
public function __construct(private readonly VCenterService $vCenter)
{
}
/**
* Health-check the new VM. Throws on failure (no rollback needed — old VM
* still owns the DB).
*/
public function healthCheck(RebuildContext $ctx): void
{
if ($ctx->newVmId === null) {
throw new \RuntimeException('Rebuild swap: newVmId is null');
}
$vcenter = VCenterModel::find($ctx->vcenterId);
if (!$vcenter || !$this->vCenter->loginServer($vcenter)) {
throw new \RuntimeException(
"Rebuild swap: cannot login to vCenter #{$ctx->vcenterId}"
);
}
$detail = $this->vCenter->GetDetailVirtualMachine($ctx->newVmId);
if (($detail['error'] ?? true) === true) {
throw new \RuntimeException(
'Rebuild swap: GetDetailVirtualMachine failed — ' . json_encode($detail)
);
}
$power = $detail['data']['power_state'] ?? null;
if ($power !== 'POWERED_ON') {
throw new \RuntimeException(
"Rebuild swap: new VM not POWERED_ON (got '{$power}')"
);
}
$ipInfo = $this->vCenter->GetIPAddressVirtualMachine($ctx->newVmId);
$guestIp = (string) ($ipInfo['data']['ip_address'] ?? '');
if ($guestIp === '') {
throw new \RuntimeException('Rebuild swap: new VM reports no guest IP');
}
// Soft check — guest may take a moment to settle on the new IP, but
// it should at least be reporting *something* on the right subnet.
Log::info('VmRebuildSwapService: health check passed', [
'module_vmware_id' => $ctx->moduleVmwareId,
'new_vm_id' => $ctx->newVmId,
'guest_ip' => $guestIp,
'expected_ip' => $ctx->ip,
'power_state' => $power,
]);
}
/**
* Atomically swap DB references:
* - module_vmware.vmware_id → new vSphere id
* - module_vmware.os_id → new OS id
* - virtual_machines.vm_id → new vSphere id (if a row exists)
*
* Preserved: services row (incl. vm_id FK), order_item_id, customer_id,
* ModuleIpAddress, ServiceAddons.
*/
public function swapDatabaseReferences(RebuildContext $ctx): void
{
if ($ctx->newVmId === null) {
throw new \RuntimeException('Rebuild swap: newVmId is null');
}
DB::transaction(function () use ($ctx) {
ModuleVmware::where('id', $ctx->moduleVmwareId)->update([
'vmware_id' => $ctx->newVmId,
'os_id' => $ctx->newOsId,
'status' => 1,
]);
VirtualMachines::where('vm_id', $ctx->oldVmId)->update([
'vm_id' => $ctx->newVmId,
'ip' => $ctx->ip,
]);
if ($ctx->serviceId) {
// Keep the existing Service intact — just make sure it stays
// marked active and its server_name reflects the VM.
Service::where('id', $ctx->serviceId)->update([
'status' => 'active',
]);
}
});
Log::info('VmRebuildSwapService: DB references swapped', [
'module_vmware_id' => $ctx->moduleVmwareId,
'old_vm_id' => $ctx->oldVmId,
'new_vm_id' => $ctx->newVmId,
]);
}
/**
* Delete the old VM from vCenter. Best-effort: the DB is already swapped,
* so a leftover ghost VM is recoverable manually and must not throw past
* the queue.
*/
public function deleteOldVm(RebuildContext $ctx): void
{
if ($ctx->oldVmId === '') {
return;
}
try {
$vcenter = VCenterModel::find($ctx->vcenterId);
if (!$vcenter || !$this->vCenter->loginServer($vcenter)) {
Log::error('VmRebuildSwapService: cannot login to delete old VM', [
'old_vm_id' => $ctx->oldVmId,
]);
return;
}
// Force power-off before delete to avoid "VM is running" rejection.
$this->vCenter->PowerOffVirtualMachine($ctx->oldVmId);
$res = $this->vCenter->DeleteVirtualMachine($ctx->oldVmId);
Log::info('VmRebuildSwapService: old VM deleted', [
'old_vm_id' => $ctx->oldVmId,
'result' => $res,
]);
} catch (Throwable $e) {
Log::error('VmRebuildSwapService: deleteOldVm threw', [
'old_vm_id' => $ctx->oldVmId,
'error' => $e->getMessage(),
]);
}
}
}
+102 -62
View File
@@ -59,17 +59,24 @@ final class VsphereHostSshService
: self::SCRIPT_LINUX;
$powerCliVmId = $this->powerCliVmId($vmId);
// Remote Windows cmd.exe command — all args double-quoted
$remoteCmd = sprintf(
'cd %s && powershell -ExecutionPolicy Bypass -NonInteractive -File %s %s %s %s %s %s',
self::SCRIPT_DIR,
$script,
$this->warg($powerCliVmId),
$this->warg($newPassword),
$this->warg($vcenterHost),
$this->warg($vcenterUser),
$this->warg($vcenterPassword),
);
$remoteCmd = strtoupper($osFamily) === VmProvisionService::OS_WINDOWS
? $this->windowsPowerShellCommand(self::SCRIPT_DIR, $script, [
$powerCliVmId,
$newPassword,
$vcenterHost,
$vcenterUser,
$vcenterPassword,
])
: sprintf(
'cd %s && powershell -ExecutionPolicy Bypass -NonInteractive -File %s %s %s %s %s %s',
self::SCRIPT_DIR,
$script,
$this->warg($powerCliVmId),
$this->warg($newPassword),
$this->warg($vcenterHost),
$this->warg($vcenterUser),
$this->warg($vcenterPassword),
);
Log::info('VsphereHostSshService: executing script', [
'vm_id' => $vmId,
@@ -93,8 +100,8 @@ final class VsphereHostSshService
* Assign a static IP to the VM guest OS via PS1 script.
* Calls assignIPLinux.ps1 or assignIPWindows.ps1 in C:\CLISave\AssignIP\.
*
* Windows script params (7): VMName IP MaskBits Gateway DNS GuestUser GuestPassword
* Linux script params (10): VCenter VCUser VCPass VMName IP MaskBits Gateway DNS GuestUser GuestPassword
* Script params:
* VmId IP Gateway Cidr VCenterHost VCenterUser VCenterPassword GuestPassword
*
* Script output: JSON {"status":"ok"} or {"status":"error","message":"..."}
*/
@@ -115,21 +122,30 @@ final class VsphereHostSshService
$script = $isWindows ? self::SCRIPT_ASSIGN_WINDOWS : self::SCRIPT_ASSIGN_LINUX;
$powerCliVmId = $this->powerCliVmId($vmId);
// assignIP{Linux|Windows}.ps1:
// VmId IP Gateway Cidr VCenterHost VCenterUser VCenterPassword GuestPassword
$remoteCmd = sprintf(
'cd %s && powershell -ExecutionPolicy Bypass -NonInteractive -File %s %s %s %s %s %s %s %s %s',
self::ASSIGN_DIR,
$script,
$this->warg($powerCliVmId),
$this->warg($ip),
$this->warg($gateway),
$this->warg($cidr),
$this->warg($vcenterHost),
$this->warg($vcenterUser),
$this->warg($vcenterPassword),
$this->warg($guestPassword),
);
$remoteCmd = $isWindows
? $this->windowsPowerShellCommand(self::ASSIGN_DIR, $script, [
$powerCliVmId,
$ip,
$gateway,
$cidr,
$vcenterHost,
$vcenterUser,
$vcenterPassword,
$guestPassword,
])
: sprintf(
'cd %s && powershell -ExecutionPolicy Bypass -NonInteractive -File %s %s %s %s %s %s %s %s %s',
self::ASSIGN_DIR,
$script,
$this->warg($powerCliVmId),
$this->warg($ip),
$this->warg($gateway),
$this->warg($cidr),
$this->warg($vcenterHost),
$this->warg($vcenterUser),
$this->warg($vcenterPassword),
$this->warg($guestPassword),
);
Log::info('VsphereHostSshService: executing assignIp', [
'vm_id' => $vmId,
@@ -154,8 +170,8 @@ final class VsphereHostSshService
* Verify that an IP is active inside the VM guest OS.
* Calls verifyIPLinux.ps1 or verifyIPWindows.ps1 in C:\CLISave\AssignIP\.
*
* Windows script params (4): VMName IP GuestUser GuestPassword
* Linux script params (7): VCenter VCUser VCPass VMName IP GuestUser GuestPassword
* Script params:
* VmId IP VCenterHost VCenterUser VCenterPassword GuestPassword
*
* Script output: JSON {"status":"ok"} or {"status":"error","message":"..."}
*/
@@ -173,19 +189,26 @@ final class VsphereHostSshService
$script = $isWindows ? self::SCRIPT_VERIFY_WINDOWS : self::SCRIPT_VERIFY_LINUX;
$powerCliVmId = $this->powerCliVmId($vmId);
// verifyIP{Linux|Windows}.ps1:
// VmId IP VCenterHost VCenterUser VCenterPassword GuestPassword
$remoteCmd = sprintf(
'cd %s && powershell -ExecutionPolicy Bypass -NonInteractive -File %s %s %s %s %s %s %s',
self::ASSIGN_DIR,
$script,
$this->warg($powerCliVmId),
$this->warg($ip),
$this->warg($vcenterHost),
$this->warg($vcenterUser),
$this->warg($vcenterPassword),
$this->warg($guestPassword),
);
$remoteCmd = $isWindows
? $this->windowsPowerShellCommand(self::ASSIGN_DIR, $script, [
$powerCliVmId,
$ip,
$vcenterHost,
$vcenterUser,
$vcenterPassword,
$guestPassword,
])
: sprintf(
'cd %s && powershell -ExecutionPolicy Bypass -NonInteractive -File %s %s %s %s %s %s %s',
self::ASSIGN_DIR,
$script,
$this->warg($powerCliVmId),
$this->warg($ip),
$this->warg($vcenterHost),
$this->warg($vcenterUser),
$this->warg($vcenterPassword),
$this->warg($guestPassword),
);
Log::info('VsphereHostSshService: executing verifyIpInGuest', [
'vm_id' => $vmId,
@@ -301,17 +324,26 @@ final class VsphereHostSshService
: self::SCRIPT_CHANGE_PWD_LINUX;
$powerCliVmId = $this->powerCliVmId($vmId);
$remoteCmd = sprintf(
'cd %s && powershell -ExecutionPolicy Bypass -NonInteractive -File %s %s %s %s %s %s %s',
self::CHANGE_PWD_DIR,
$script,
$this->warg($powerCliVmId),
$this->warg($newPassword),
$this->warg($vcenterHost),
$this->warg($vcenterUser),
$this->warg($vcenterPassword),
$this->warg($currentGuestPassword),
);
$remoteCmd = strtoupper($osFamily) === VmProvisionService::OS_WINDOWS
? $this->windowsPowerShellCommand(self::CHANGE_PWD_DIR, $script, [
$powerCliVmId,
$newPassword,
$vcenterHost,
$vcenterUser,
$vcenterPassword,
$currentGuestPassword,
])
: sprintf(
'cd %s && powershell -ExecutionPolicy Bypass -NonInteractive -File %s %s %s %s %s %s %s',
self::CHANGE_PWD_DIR,
$script,
$this->warg($powerCliVmId),
$this->warg($newPassword),
$this->warg($vcenterHost),
$this->warg($vcenterUser),
$this->warg($vcenterPassword),
$this->warg($currentGuestPassword),
);
Log::info('VsphereHostSshService: changePasswordCustomer', [
'vm_id' => $vmId,
@@ -331,6 +363,22 @@ final class VsphereHostSshService
// ─────────────────────────────────────────────────────────────────────────
/**
* @param array<int, string> $arguments
*/
private function windowsPowerShellCommand(string $workingDirectory, string $script, array $arguments): string
{
return sprintf(
'cd %s && powershell -ExecutionPolicy Bypass -File %s %s',
$workingDirectory,
$script,
implode(' ', array_map(
fn (string $argument): string => $this->warg($argument),
$arguments,
)),
);
}
/**
* Windows cmd.exe safe argument: wrap in double quotes, escape inner double quotes.
*/
@@ -348,12 +396,4 @@ final class VsphereHostSshService
return $vmId;
}
/**
* Single-quote wrap for passwords — prevents PowerShell variable expansion on $, %, ^ etc.
* Inside a single-quoted PS string, a literal single quote is escaped as ''.
*/
private function sarg(string $value): string
{
return "'" . str_replace("'", "''", $value) . "'";
}
}
@@ -0,0 +1,59 @@
<?php
declare(strict_types=1);
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
// Broaden the status enum to a free-form string so the rebuild
// pipeline can use 'pending'/'in_progress'/'completed'/'failed'.
DB::statement("ALTER TABLE reinstall_logs MODIFY COLUMN status VARCHAR(32) NOT NULL DEFAULT 'pending'");
Schema::table('reinstall_logs', function (Blueprint $table) {
$table->unsignedBigInteger('service_id')->nullable()->after('custome_id');
$table->string('old_vmware_id')->nullable()->after('template_id');
$table->string('new_vmware_id')->nullable()->after('old_vmware_id');
// Plaintext + encrypted copy of the credential applied to the
// new VM. Encrypted column uses SignRSA so it matches the
// existing module_vmware.password format.
$table->text('password')->nullable()->after('ip_address');
$table->text('plain_password')->nullable()->after('password');
$table->boolean('password_supplied_by_customer')->default(false)->after('plain_password');
$table->timestamp('started_at')->nullable()->after('error_message');
$table->timestamp('completed_at')->nullable()->after('started_at');
$table->index('service_id');
$table->index('custome_id');
$table->index('status');
});
}
public function down(): void
{
Schema::table('reinstall_logs', function (Blueprint $table) {
$table->dropIndex(['service_id']);
$table->dropIndex(['custome_id']);
$table->dropIndex(['status']);
$table->dropColumn([
'service_id',
'old_vmware_id',
'new_vmware_id',
'password',
'plain_password',
'password_supplied_by_customer',
'started_at',
'completed_at',
]);
});
DB::statement(
"ALTER TABLE reinstall_logs MODIFY COLUMN status ENUM('pending','success','failed') NOT NULL DEFAULT 'pending'"
);
}
};
+63 -35
View File
@@ -1,48 +1,76 @@
# assignIPWindows.ps1
# Deploy to: C:\CLISave\AssignIP\assignIPWindows.ps1 on the vSphere Windows host
#
# Usage: powershell -ExecutionPolicy Bypass -NonInteractive -File assignIPWindows.ps1 \
# <vm_id> <ip> <gateway> <cidr> <vcenter_host> <vcenter_user> <vcenter_password> <guest_password>
#
# Output: SUCCESS | FAIL (exactly one line)
param(
[string]$VMName,
[string]$IP,
[int]$MaskBits,
[string]$Gateway,
[string]$DNS,
[string]$GuestUser,
[string]$GuestPassword
[Parameter(Mandatory = $true)][string]$VmId,
[Parameter(Mandatory = $true)][string]$Ip,
[Parameter(Mandatory = $true)][string]$Gateway,
[Parameter(Mandatory = $true)][string]$Cidr,
[Parameter(Mandatory = $true)][string]$VCenterHost,
[Parameter(Mandatory = $true)][string]$VCenterUser,
[Parameter(Mandatory = $true)][string]$VCenterPassword,
[Parameter(Mandatory = $true)][string]$GuestPassword
)
$sec = ConvertTo-SecureString $GuestPassword -AsPlainText -Force
$cred = New-Object pscredential($GuestUser, $sec)
Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"
try {
$vm = Get-VM -Name $VMName -ErrorAction Stop
Import-Module VMware.VimAutomation.Core -ErrorAction Stop
if ($vm.PowerState -ne 'PoweredOn') {
[pscustomobject]@{ status="error"; message="VM is powered off" } | ConvertTo-Json
exit
}
Set-PowerCLIConfiguration -InvalidCertificateAction Ignore -Confirm:$false | Out-Null
Set-PowerCLIConfiguration -Scope User -ParticipateInCEIP $false -Confirm:$false | Out-Null
if ($vm.ExtensionData.Guest.ToolsStatus -notin @('toolsOk','toolsOld')) {
[pscustomobject]@{ status="error"; message="VMware Tools not ready" } | ConvertTo-Json
exit
}
Connect-VIServer -Server $VCenterHost -User $VCenterUser -Password $VCenterPassword -Force -ErrorAction Stop | Out-Null
$script = @"
`$adapter = Get-NetAdapter | Where-Object { `$_.Status -eq 'Up' } | Select-Object -First 1
Remove-NetIPAddress -InterfaceIndex `$adapter.ifIndex -Confirm:`$false -ErrorAction SilentlyContinue
Remove-NetRoute -InterfaceIndex `$adapter.ifIndex -Confirm:`$false -ErrorAction SilentlyContinue
New-NetIPAddress -InterfaceIndex `$adapter.ifIndex -IPAddress "$IP" -PrefixLength $MaskBits -DefaultGateway "$Gateway"
Set-DnsClientServerAddress -InterfaceIndex `$adapter.ifIndex -ServerAddresses "$DNS"
$vm = Get-VM -Id "VirtualMachine-$VmId" -ErrorAction Stop
# PowerShell script executed inside the Windows guest via VMware Tools
$guestScript = @"
$ErrorActionPreference = 'Stop'
try {
# Find the first active ethernet adapter that is not a loopback
$adapter = Get-NetAdapter |
Where-Object { $_.Status -eq 'Up' -and $_.InterfaceDescription -notlike '*Loopback*' } |
Sort-Object -Property InterfaceIndex |
Select-Object -First 1
if (-not $adapter) {
throw 'No active network adapter found'
}
# Remove existing IPv4 addresses and routes on this interface
$adapter |
Get-NetIPAddress -AddressFamily IPv4 -ErrorAction SilentlyContinue |
Remove-NetIPAddress -Confirm:$false -ErrorAction SilentlyContinue
Get-NetRoute -InterfaceIndex $adapter.InterfaceIndex -AddressFamily IPv4 -ErrorAction SilentlyContinue |
Remove-NetRoute -Confirm:$false -ErrorAction SilentlyContinue
# Set new static IP
New-NetIPAddress -InterfaceIndex $adapter.InterfaceIndex -IPAddress '$Ip' -PrefixLength $Cidr -DefaultGateway '$Gateway' -ErrorAction Stop | Out-Null
Write-Output 'SUCCESS'
} catch {
Write-Output 'FAIL'
}
"@
$res = Invoke-VMScript -VM $vm -ScriptText $script -GuestCredential $cred -ScriptType PowerShell
[pscustomobject]@{
status="ok"
message="IP updated"
ip=$IP
} | ConvertTo-Json
$result = Invoke-VMScript -VM $vm -ScriptText $guestScript -GuestUser "administrator" -GuestPassword $GuestPassword -ScriptType Powershell -ErrorAction Stop
if ($result.ScriptOutput -match "SUCCESS") {
Write-Output "SUCCESS"
} else {
Write-Output "FAIL"
}
} catch {
[pscustomobject]@{
status="error"
message=$_.Exception.Message
} | ConvertTo-Json
Write-Output "FAIL"
Write-Output $_.Exception.Message
} finally {
Disconnect-VIServer -Confirm:$false -ErrorAction SilentlyContinue | Out-Null
}
+1
View File
@@ -102,6 +102,7 @@ echo SUCCESS
}
} catch {
Write-Output "FAIL"
Write-Output $_.Exception.Message
} finally {
Disconnect-VIServer -Confirm:$false -ErrorAction SilentlyContinue | Out-Null
}
+16 -49
View File
@@ -1,11 +1,4 @@
# assignIPWindows.ps1
# Deploy to: C:\CLISave\AssignIP\assignIPWindows.ps1 on the vSphere Windows host
#
# Usage: powershell -ExecutionPolicy Bypass -NonInteractive -File assignIPWindows.ps1 \
# <vm_id> <ip> <gateway> <cidr> <vcenter_host> <vcenter_user> <vcenter_password> <guest_password>
#
# Output: SUCCESS | FAIL (exactly one line)
param(
[Parameter(Mandatory=$true)][string]$VmId,
[Parameter(Mandatory=$true)][string]$Ip,
@@ -17,6 +10,21 @@ param(
[Parameter(Mandatory=$true)][string]$GuestPassword
)
$guestScript = @"
`$ErrorActionPreference = 'Stop'
try {
`$adapter = Get-NetAdapter | Where-Object { `$_.Status -eq 'Up' -and `$_.InterfaceDescription -notlike '*Loopback*' } | Sort-Object InterfaceIndex | Select-Object -First 1
if (-not `$adapter) { throw 'No active network adapter found' }
Get-NetIPAddress -InterfaceIndex `$adapter.InterfaceIndex -AddressFamily IPv4 -ErrorAction SilentlyContinue | Remove-NetIPAddress -Confirm:`$false -ErrorAction SilentlyContinue
Get-NetRoute -InterfaceIndex `$adapter.InterfaceIndex -AddressFamily IPv4 -ErrorAction SilentlyContinue | Remove-NetRoute -Confirm:`$false -ErrorAction SilentlyContinue
New-NetIPAddress -InterfaceIndex `$adapter.InterfaceIndex -IPAddress '$Ip' -PrefixLength $Cidr -DefaultGateway '$Gateway' -ErrorAction Stop | Out-Null
Write-Output 'SUCCESS'
}
catch {
Write-Output "FAIL: `$(`$PSItem.Exception.Message)"
}
"@
Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"
@@ -24,50 +32,9 @@ try {
Import-Module VMware.VimAutomation.Core -ErrorAction Stop
Set-PowerCLIConfiguration -InvalidCertificateAction Ignore -Confirm:$false | Out-Null
Set-PowerCLIConfiguration -Scope User -ParticipateInCEIP $false -Confirm:$false | Out-Null
Connect-VIServer -Server $VCenterHost -User $VCenterUser -Password $VCenterPassword -Force -ErrorAction Stop | Out-Null
$vm = Get-VM -Id "VirtualMachine-$VmId" -ErrorAction Stop
# PowerShell script executed inside the Windows guest via VMware Tools
$guestScript = @"
`$ErrorActionPreference = 'Stop'
try {
# Find the first active ethernet adapter that is not a loopback
`$adapter = Get-NetAdapter |
Where-Object { `$_.Status -eq 'Up' -and `$_.InterfaceDescription -notlike '*Loopback*' } |
Sort-Object -Property InterfaceIndex |
Select-Object -First 1
if (-not `$adapter) { throw 'No active network adapter found' }
# Remove existing IPv4 addresses and routes on this interface
`$adapter | Get-NetIPAddress -AddressFamily IPv4 -ErrorAction SilentlyContinue |
Remove-NetIPAddress -Confirm:`$false -ErrorAction SilentlyContinue
Get-NetRoute -InterfaceIndex `$adapter.InterfaceIndex -AddressFamily IPv4 -ErrorAction SilentlyContinue |
Remove-NetRoute -Confirm:`$false -ErrorAction SilentlyContinue
# Set new static IP
New-NetIPAddress `
-InterfaceIndex `$adapter.InterfaceIndex `
-IPAddress '$Ip' `
-PrefixLength $Cidr `
-DefaultGateway '$Gateway' `
-ErrorAction Stop | Out-Null
Write-Output 'SUCCESS'
} catch {
Write-Output 'FAIL'
}
"@
$result = Invoke-VMScript -VM $vm `
-ScriptText $guestScript `
-GuestUser "administrator" `
-GuestPassword $GuestPassword `
-ScriptType Powershell `
-ErrorAction Stop
$result = Invoke-VMScript -VM $vm -ScriptText $guestScript -GuestUser "administrator" -GuestPassword $GuestPassword -ScriptType Powershell -ErrorAction Stop
if ($result.ScriptOutput -match "SUCCESS") {
Write-Output "SUCCESS"
} else {
+10
View File
@@ -27,6 +27,7 @@ use App\Http\Controllers\Api\Admin\Bank\BankController;
use App\Http\Controllers\Api\Admin\EmailTemplate\EmailTemplateController;
use App\Http\Controllers\Api\Admin\EmailNotificationTemplateController;
use App\Http\Controllers\Api\Admin\ActivityLog\ActivityLogController;
use App\Http\Controllers\Api\Admin\Rebuild\AdminReinstallLogController;
use App\Http\Controllers\Api\Admin\Transaction\AdminTransactionController;
use App\Http\Controllers\Api\Admin\Ticket\TicketController as AdminTicketController;
use App\Http\Controllers\Api\Admin\Docs\PostCategoryController as AdminPostCategoryController;
@@ -506,6 +507,15 @@ Route::prefix('v1')
Route::get('stats', 'stats');
Route::get('{id}', 'show');
});
// VM Rebuild Logs — who rebuilt, which OS, post-rebuild password
Route::prefix('rebuild-logs')
->controller(AdminReinstallLogController::class)
->group(function () {
Route::get('/', 'index')->middleware('perm:vps.view');
Route::get('stats', 'stats')->middleware('perm:vps.view');
Route::get('{id}', 'show')->middleware('perm:vps.view')->whereNumber('id');
});
});
});