From ea771a701317c06f0037016e94e15e0fe7106a31 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 26 May 2026 13:42:03 +0530 Subject: [PATCH 01/28] Fix Instance Backup related events (#13180) --- ...signVirtualMachineToBackupOfferingCmd.java | 11 +++ .../command/user/backup/CreateBackupCmd.java | 7 +- .../user/backup/CreateBackupScheduleCmd.java | 13 ++- ...veVirtualMachineFromBackupOfferingCmd.java | 11 +++ ...storeVolumeFromBackupAndAttachToVMCmd.java | 11 +++ .../cloudstack/backup/NASBackupProvider.java | 8 +- .../cloudstack/backup/BackupManagerImpl.java | 97 +++++++++---------- .../cloudstack/backup/BackupManagerTest.java | 6 +- 8 files changed, 104 insertions(+), 60 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/AssignVirtualMachineToBackupOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/AssignVirtualMachineToBackupOfferingCmd.java index 28cd642e1a41..29b1e740b67e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/AssignVirtualMachineToBackupOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/AssignVirtualMachineToBackupOfferingCmd.java @@ -21,6 +21,7 @@ import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; @@ -102,6 +103,16 @@ public long getEntityOwnerId() { return CallContext.current().getCallingAccount().getId(); } + @Override + public Long getApiResourceId() { + return vmId; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } + @Override public String getEventType() { return EventTypes.EVENT_VM_BACKUP_OFFERING_ASSIGN; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java index ca60ea674fe3..8a4053aa15da 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java @@ -123,7 +123,12 @@ public void execute() throws ResourceUnavailableException, InsufficientCapacityE @Override public ApiCommandResourceType getApiResourceType() { - return ApiCommandResourceType.Backup; + return ApiCommandResourceType.VirtualMachine; + } + + @Override + public Long getApiResourceId() { + return vmId; } @Override diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java index 67ad7c71503f..f6e17a2b3908 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java @@ -21,6 +21,7 @@ import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseCmd; @@ -81,7 +82,7 @@ public class CreateBackupScheduleCmd extends BaseCmd { @Parameter(name = ApiConstants.QUIESCE_VM, type = CommandType.BOOLEAN, required = false, - description = "Quiesce the instance before checkpointing the disks for backup. Applicable only to NAS backup provider. " + + description = "Quiesce the Instance before checkpointing the disks for backup. Applicable only to NAS backup provider. " + "The filesystem is frozen before the backup starts and thawed immediately after. " + "Requires the instance to have the QEMU Guest Agent installed and running.", since = "4.21.0") @@ -139,4 +140,14 @@ public void execute() throws ServerApiException { public long getEntityOwnerId() { return CallContext.current().getCallingAccount().getId(); } + + @Override + public Long getApiResourceId() { + return vmId; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RemoveVirtualMachineFromBackupOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RemoveVirtualMachineFromBackupOfferingCmd.java index dcf9f15b4dc5..62c2f03d8807 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RemoveVirtualMachineFromBackupOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RemoveVirtualMachineFromBackupOfferingCmd.java @@ -21,6 +21,7 @@ import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; @@ -99,6 +100,16 @@ public long getEntityOwnerId() { return CallContext.current().getCallingAccount().getId(); } + @Override + public Long getApiResourceId() { + return vmId; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } + @Override public String getEventType() { return EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVE; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java index 4644687817df..c15e6f8de684 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java @@ -22,6 +22,7 @@ import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; @@ -127,4 +128,14 @@ public String getEventType() { public String getEventDescription() { return "Restoring volume "+ volumeUuid + " from backup " + getResourceUuid(ApiConstants.BACKUP_ID) + " and attaching it to Instance " + getResourceUuid(ApiConstants.VIRTUAL_MACHINE_ID); } + + @Override + public Long getApiResourceId() { + return vmId; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } } diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index df9336026f4d..a7121f149e30 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -226,12 +226,12 @@ public Pair takeBackup(final VirtualMachine vm, Boolean quiesce } catch (AgentUnavailableException e) { logger.error("Unable to contact backend control plane to initiate backup for VM {}", vm.getInstanceName()); backupVO.setStatus(Backup.Status.Failed); - backupDao.remove(backupVO.getId()); + backupDao.update(backupVO.getId(), backupVO); throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); } catch (OperationTimedoutException e) { logger.error("Operation to initiate backup timed out for VM {}", vm.getInstanceName()); backupVO.setStatus(Backup.Status.Failed); - backupDao.remove(backupVO.getId()); + backupDao.update(backupVO.getId(), backupVO); throw new CloudRuntimeException("Operation to initiate backup timed out, please try again"); } @@ -249,12 +249,12 @@ public Pair takeBackup(final VirtualMachine vm, Boolean quiesce } else { logger.error("Failed to take backup for VM {}: {}", vm.getInstanceName(), answer != null ? answer.getDetails() : "No answer received"); if (answer.getNeedsCleanup()) { - logger.error("Backup cleanup failed for VM {}. Leaving the backup in Error state.", vm.getInstanceName()); + logger.error("Backup cleanup failed for VM {}. Leaving the backup in Error state. Backup should be manually deleted to free up the space", vm.getInstanceName()); backupVO.setStatus(Backup.Status.Error); backupDao.update(backupVO.getId(), backupVO); } else { backupVO.setStatus(Backup.Status.Failed); - backupDao.remove(backupVO.getId()); + backupDao.update(backupVO.getId(), backupVO); } return new Pair<>(false, null); } diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index fd4fc9466066..1cbb668206d9 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -576,12 +576,12 @@ public String createVolumeInfoFromVolumes(List vmVolumes) { } @Override - @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_OFFERING_ASSIGN, eventDescription = "assign VM to backup offering", async = true) + @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_OFFERING_ASSIGN, eventDescription = "assign Instance to Backup Offering", async = true) public boolean assignVMToBackupOffering(Long vmId, Long offeringId) { final VMInstanceVO vm = findVmById(vmId); if (!Arrays.asList(VirtualMachine.State.Running, VirtualMachine.State.Stopped, VirtualMachine.State.Shutdown).contains(vm.getState())) { - throw new CloudRuntimeException("VM is not in running or stopped state"); + throw new CloudRuntimeException("Instance is not in running or stopped state"); } validateBackupForZone(vm.getDataCenterId()); @@ -589,7 +589,7 @@ public boolean assignVMToBackupOffering(Long vmId, Long offeringId) { accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); if (vm.getBackupOfferingId() != null) { - throw new CloudRuntimeException("VM already is assigned to a backup offering, please remove the VM from its previous offering"); + throw new CloudRuntimeException("Instance already is assigned to a backup offering, please remove the Instance from its previous offering"); } final BackupOfferingVO offering = backupOfferingDao.findById(offeringId); @@ -621,12 +621,12 @@ public VMInstanceVO doInTransaction(final TransactionStatus status) { vm.setBackupVolumes(createVolumeInfoFromVolumes(new ArrayList<>(volumeDao.findByInstance(vmId)))); if (!backupProvider.assignVMToBackupOffering(vm, offering)) { - throw new CloudRuntimeException("Failed to assign the VM to the backup offering, please try removing the assignment and try again."); + throw new CloudRuntimeException("Failed to assign the Instance to the Backup Offering, please try removing the assignment and try again."); } if (!vmInstanceDao.update(vmId, vm)) { backupProvider.removeVMFromBackupOffering(vm); - throw new CloudRuntimeException("Failed to update VM assignment to the backup offering in the DB, please try again."); + throw new CloudRuntimeException("Failed to update Instance assignment to the Backup Offering in the DB, please try again."); } UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_OFFERING_ASSIGN, vm.getAccountId(), vm.getDataCenterId(), vmId, @@ -635,7 +635,7 @@ public VMInstanceVO doInTransaction(final TransactionStatus status) { "uuid", "instanceName", "backupOfferingId", "backupVolumes"), ReflectionToStringBuilderUtils.reflectOnlySelectedFields(offering, "uuid", "name", "externalId", "provider"))); } catch (Exception e) { - String msg = String.format("Failed to assign VM [%s] to the Backup Offering [%s], using provider [name: %s, class: %s], due to: [%s].", + String msg = String.format("Failed to assign Instance [%s] to the Backup Offering [%s], using provider [name: %s, class: %s], due to: [%s].", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(vm, "uuid", "instanceName", "backupOfferingId", "backupVolumes"), ReflectionToStringBuilderUtils.reflectOnlySelectedFields(offering, "uuid", "name", "externalId", "provider"), backupProvider.getName(), backupProvider.getClass().getSimpleName(), e.getMessage()); @@ -649,11 +649,11 @@ public VMInstanceVO doInTransaction(final TransactionStatus status) { } @Override - @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVE, eventDescription = "remove VM from backup offering", async = true) + @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVE, eventDescription = "remove Instance from Backup Offering", async = true) public boolean removeVMFromBackupOffering(final Long vmId, final boolean forced) { final VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(vmId); if (vm == null) { - throw new CloudRuntimeException(String.format("Can't find any VM with ID: [%s].", vmId)); + throw new CloudRuntimeException(String.format("Can't find any Instance with ID: [%s].", vmId)); } validateBackupForZone(vm.getDataCenterId()); @@ -670,8 +670,8 @@ public boolean removeVMFromBackupOffering(final Long vmId, final boolean forced) } if (!forced && backupProvider.willDeleteBackupsOnOfferingRemoval()) { - String message = String.format("To remove VM [id: %s, name: %s] from Backup Offering [id: %s, name: %s] using the provider [%s], please specify the " - + "forced:true option to allow the deletion of all jobs and backups for this VM or remove the backups that this VM has with the backup " + String message = String.format("To remove Instance [id: %s, name: %s] from Backup Offering [id: %s, name: %s] using the provider [%s], please specify the " + + "forced:true option to allow the deletion of all jobs and backups for this Instance or remove the backups that this Instance has with the backup " + "offering.", vm.getUuid(), vm.getInstanceName(), offering.getUuid(), offering.getName(), backupProvider.getClass().getSimpleName()); throw new CloudRuntimeException(message); } @@ -709,7 +709,7 @@ public boolean removeVMFromBackupOffering(final Long vmId, final boolean forced) } @Override - @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_SCHEDULE_CONFIGURE, eventDescription = "configuring VM backup schedule") + @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_SCHEDULE_CONFIGURE, eventDescription = "configuring Instance Backup Schedule") public BackupSchedule configureBackupSchedule(CreateBackupScheduleCmd cmd) { final Long vmId = cmd.getVmId(); final DateUtil.IntervalType intervalType = cmd.getIntervalType(); @@ -725,7 +725,7 @@ public BackupSchedule configureBackupSchedule(CreateBackupScheduleCmd cmd) { accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); if (vm.getBackupOfferingId() == null) { - throw new CloudRuntimeException("Cannot configure backup schedule for the VM without having any backup offering"); + throw new CloudRuntimeException("Cannot configure Backup Schedule for the Instance as it is not assigned any Backup Offering"); } final BackupOffering offering = backupOfferingDao.findById(vm.getBackupOfferingId()); @@ -859,7 +859,7 @@ public List listBackupSchedules(ListBackupScheduleCmd cmd) { } @Override - @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_SCHEDULE_DELETE, eventDescription = "deleting VM backup schedule") + @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_SCHEDULE_DELETE, eventDescription = "deleting Instance Backup Schedule") public boolean deleteBackupSchedule(DeleteBackupScheduleCmd cmd) { Long vmId = cmd.getVmId(); Long id = cmd.getId(); @@ -910,7 +910,7 @@ protected boolean deleteAllVmBackupSchedules(long vmId) { } @Override - @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_CREATE, eventDescription = "creating VM backup", async = true) + @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_CREATE, eventDescription = "creating Instance Backup", async = true) public boolean createBackup(CreateBackupCmd cmd, Object job) throws ResourceAllocationException { Long vmId = cmd.getVmId(); Account caller = CallContext.current().getCallingAccount(); @@ -920,17 +920,17 @@ public boolean createBackup(CreateBackupCmd cmd, Object job) throws ResourceAllo accountManager.checkAccess(caller, null, true, vm); if (vm.getBackupOfferingId() == null) { - throw new CloudRuntimeException("VM has not backup offering configured, cannot create backup before assigning it to a backup offering"); + throw new CloudRuntimeException("Cannot create backup as the Instance doesn't have a Backup Offering assigned"); } final BackupOffering offering = backupOfferingDao.findById(vm.getBackupOfferingId()); if (offering == null) { - throw new CloudRuntimeException("VM backup offering not found"); + throw new CloudRuntimeException("Instance Backup Offering not found"); } final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); if (backupProvider == null) { - throw new CloudRuntimeException("VM backup provider not found for the offering"); + throw new CloudRuntimeException("Instance backup provider not found for the Offering"); } if (!offering.isUserDrivenBackupAllowed()) { @@ -970,14 +970,9 @@ private void createCheckedBackup(CreateBackupCmd cmd, Account owner, boolean isS CheckedReservation backupStorageReservation = new CheckedReservation(owner, Resource.ResourceType.backup_storage, backupSize, reservationDao, resourceLimitMgr)) { - ActionEventUtils.onStartedActionEvent(User.UID_SYSTEM, vm.getAccountId(), - EventTypes.EVENT_VM_BACKUP_CREATE, "creating backup for VM ID:" + vm.getUuid(), - vmId, ApiCommandResourceType.VirtualMachine.toString(), - true, 0); - Pair result = backupProvider.takeBackup(vm, cmd.getQuiesceVM()); if (!result.first()) { - throw new CloudRuntimeException("Failed to create VM backup"); + throw new CloudRuntimeException("Failed to create Instance Backup"); } Backup backup = result.second(); if (backup != null) { @@ -1079,7 +1074,7 @@ protected void deleteExcessBackups(List backups, int amountOfBackupsTo for (int i = 0; i < amountOfBackupsToDelete; i++) { BackupVO backup = backups.get(i); if (deleteBackup(backup.getId(), false)) { - String eventDescription = String.format("Successfully deleted backup for VM [ID: %s], suiting the retention specified in the backup schedule [ID: %s]", backup.getVmId(), backupScheduleId); + String eventDescription = String.format("Successfully deleted Backup for Instance [ID: %s], suiting the retention specified in the Backup Schedule [ID: %s]", backup.getVmId(), backupScheduleId); logger.info(eventDescription); ActionEventUtils.onCompletedActionEvent( User.UID_SYSTEM, backup.getAccountId(), EventVO.LEVEL_INFO, @@ -1176,9 +1171,9 @@ public boolean importRestoredVM(long zoneId, long domainId, long accountId, long logger.error(String.format("Failed to import VM [vmInternalName: %s] from backup restoration [%s] with hypervisor [type: %s] due to: [%s].", vmInternalName, ReflectionToStringBuilderUtils.reflectOnlySelectedFields(backup, "id", "uuid", "vmId", "externalId", "backupType"), hypervisorType, e.getMessage()), e); ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_ERROR, EventTypes.EVENT_VM_BACKUP_RESTORE, - String.format("Failed to import VM %s from backup %s with hypervisor [type: %s]", vmInternalName, backup.getUuid(), hypervisorType), + String.format("Failed to import Instance %s from Backup %s with hypervisor [type: %s]", vmInternalName, backup.getUuid(), hypervisorType), vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0); - throw new CloudRuntimeException("Error during vm backup restoration and import: " + e.getMessage()); + throw new CloudRuntimeException("Error during Instance Backup restoration and import: " + e.getMessage()); } if (vm == null) { String message = String.format("Failed to import restored VM %s with hypervisor type %s using backup of VM ID %s", @@ -1188,14 +1183,14 @@ public boolean importRestoredVM(long zoneId, long domainId, long accountId, long message, vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0); } else { ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_VM_BACKUP_RESTORE, - String.format("Restored VM %s from backup %s", vm.getUuid(), backup.getUuid()), + String.format("Restored Instance %s from Backup %s", vm.getUuid(), backup.getUuid()), vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0); } return vm != null; } @Override - @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_RESTORE, eventDescription = "restoring VM from backup", async = true) + @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_RESTORE, eventDescription = "restoring Instance from Backup", async = true) public boolean restoreBackup(final Long backupId) { final BackupVO backup = backupDao.findById(backupId); if (backup == null) { @@ -1214,7 +1209,7 @@ public boolean restoreBackup(final Long backupId) { if (vm.getRemoved() == null && !vm.getState().equals(VirtualMachine.State.Stopped) && !vm.getState().equals(VirtualMachine.State.Destroyed)) { - throw new CloudRuntimeException("Existing VM should be stopped before being restored from backup"); + throw new CloudRuntimeException("Existing Instance should be stopped before being restored from Backup"); } // This is done to handle historic backups if any with Veeam / Networker plugins @@ -1222,7 +1217,7 @@ public boolean restoreBackup(final Long backupId) { vm.getBackupVolumeList() : backup.getBackedUpVolumes(); List vmVolumes = volumeDao.findByInstance(vm.getId()); if (vmVolumes.size() != backupVolumes.size()) { - throw new CloudRuntimeException("Unable to restore VM with the current backup as the backup has different number of disks as the VM"); + throw new CloudRuntimeException("Unable to restore Instance with the current Backup as the Backup has different number of disks to the Instance"); } BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(vm.getBackupOfferingId()); @@ -1256,16 +1251,16 @@ protected void tryRestoreVM(BackupVO backup, VMInstanceVO vm, BackupOffering off updateVmState(vm, VirtualMachine.Event.RestoringRequested, VirtualMachine.State.Restoring); updateVolumeState(vm, Volume.Event.RestoreRequested, Volume.State.Restoring); ActionEventUtils.onStartedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventTypes.EVENT_VM_BACKUP_RESTORE, - String.format("Restoring VM %s from backup %s", vm.getUuid(), backup.getUuid()), + String.format("Restoring Instance %s from Backup %s", vm.getUuid(), backup.getUuid()), vm.getId(), ApiCommandResourceType.VirtualMachine.toString(), true, 0); final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); if (!backupProvider.restoreVMFromBackup(vm, backup)) { ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_ERROR, EventTypes.EVENT_VM_BACKUP_RESTORE, - String.format("Failed to restore VM %s from backup %s", vm.getInstanceName(), backup.getUuid()), + String.format("Failed to restore Instance %s from Backup %s", vm.getInstanceName(), backup.getUuid()), vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0); - throw new CloudRuntimeException("Error restoring VM from backup with uuid " + backup.getUuid()); + throw new CloudRuntimeException("Error restoring Instance from Backup with uuid " + backup.getUuid()); } // The restore process is executed by a backup provider outside of ACS, I am using the catch-all (Exception) to // ensure that no provider-side exception is missed. Therefore, we have a proper handling of exceptions, and rollbacks if needed. @@ -1273,7 +1268,7 @@ protected void tryRestoreVM(BackupVO backup, VMInstanceVO vm, BackupOffering off logger.error(String.format("Failed to restore backup [%s] due to: [%s].", backupDetailsInMessage, e.getMessage()), e); updateVolumeState(vm, Volume.Event.RestoreFailed, Volume.State.Ready); updateVmState(vm, VirtualMachine.Event.RestoringFailed, VirtualMachine.State.Stopped); - throw new CloudRuntimeException(String.format("Error restoring VM from backup [%s].", backupDetailsInMessage)); + throw new CloudRuntimeException(String.format("Error restoring Instance from Backup [%s].", backupDetailsInMessage)); } } @@ -1288,10 +1283,10 @@ private void updateVmState(VMInstanceVO vm, VirtualMachine.Event event, VirtualM Transaction.execute(TransactionLegacy.CLOUD_DB, (TransactionCallback) status -> { try { if (!virtualMachineManager.stateTransitTo(vm, event, vm.getHostId())) { - throw new CloudRuntimeException(String.format("Unable to change state of VM [%s] to [%s].", vm, next)); + throw new CloudRuntimeException(String.format("Unable to change state of Instance [%s] to [%s].", vm, next)); } } catch (NoTransitionException e) { - String errMsg = String.format("Failed to update state of VM [%s] with event [%s] due to [%s].", vm, event, e.getMessage()); + String errMsg = String.format("Failed to update state of Instance [%s] with event [%s] due to [%s].", vm, event, e.getMessage()); logger.error(errMsg, e); throw new RuntimeException(errMsg); } @@ -1503,7 +1498,7 @@ public boolean restoreBackupToVM(final Long backupId, final Long vmId) throws Cl throw new CloudRuntimeException("Instance with ID " + backup.getVmId() + " couldn't be found."); } if (!vm.getState().equals(VirtualMachine.State.Stopped)) { - throw new CloudRuntimeException("The VM should be in stopped state"); + throw new CloudRuntimeException("The Instance should be in stopped state"); } List backupVolumes = backup.getBackedUpVolumes(); @@ -1567,7 +1562,7 @@ public boolean restoreBackupToVM(final Long backupId, final Long vmId) throws Cl } @Override - @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_RESTORE, eventDescription = "restoring VM from backup", async = true) + @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_RESTORE_VOLUME_TO_VM, eventDescription = "restoring Volume from Backup to Instance", async = true) public boolean restoreBackupVolumeAndAttachToVM(final String backedUpVolumeUuid, final Long backupId, final Long vmId) throws Exception { if (StringUtils.isEmpty(backedUpVolumeUuid)) { throw new CloudRuntimeException("Invalid volume ID passed"); @@ -1585,7 +1580,7 @@ public boolean restoreBackupVolumeAndAttachToVM(final String backedUpVolumeUuid, accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); if (vm.getBackupOfferingId() != null && !BackupEnableAttachDetachVolumes.value()) { - throw new CloudRuntimeException("The selected VM has backups, cannot restore and attach volume to the VM."); + throw new CloudRuntimeException("The selected Instance has backups, cannot restore and attach Volume to the Instance."); } if (backup.getZoneId() != vm.getDataCenterId()) { @@ -1596,7 +1591,7 @@ public boolean restoreBackupVolumeAndAttachToVM(final String backedUpVolumeUuid, if (volumeInfoList == null) { final VMInstanceVO vmFromBackup = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); if (vmFromBackup == null) { - throw new CloudRuntimeException("VM reference for the provided VM backup not found"); + throw new CloudRuntimeException("Instance reference for the provided Instance backup not found"); } else if (vmFromBackup == null || vmFromBackup.getBackupVolumeList() == null) { throw new CloudRuntimeException("Volumes metadata not found in the backup"); } @@ -1610,7 +1605,7 @@ public boolean restoreBackupVolumeAndAttachToVM(final String backedUpVolumeUuid, accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); final BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(backup.getBackupOfferingId()); if (offering == null) { - throw new CloudRuntimeException("Failed to find VM backup offering"); + throw new CloudRuntimeException("Failed to find Instance Backup Offering"); } BackupProvider backupProvider = getBackupProvider(offering.getProvider()); @@ -1636,12 +1631,12 @@ public boolean restoreBackupVolumeAndAttachToVM(final String backedUpVolumeUuid, Pair result = restoreBackedUpVolume(backupVolumeInfo, backup, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); if (BooleanUtils.isFalse(result.first())) { - throw new CloudRuntimeException(String.format("Error restoring volume [%s] of VM [%s] to host [%s] using backup provider [%s] due to: [%s].", + throw new CloudRuntimeException(String.format("Error restoring Volume [%s] of Instance [%s] to host [%s] using backup provider [%s] due to: [%s].", backedUpVolumeUuid, vm.getUuid(), host.getUuid(), backupProvider.getName(), result.second())); } if (!attachVolumeToVM(vm.getDataCenterId(), result.second(), backupVolumeInfo, backedUpVolumeUuid, vm, datastore.getUuid(), backup)) { - throw new CloudRuntimeException(String.format("Error attaching volume [%s] to VM [%s].", backedUpVolumeUuid, vm.getUuid())); + throw new CloudRuntimeException(String.format("Error attaching Volume [%s] to Instance [%s].", backedUpVolumeUuid, vm.getUuid())); } return true; } @@ -1670,7 +1665,7 @@ protected Pair restoreBackedUpVolume(final Backup.VolumeInfo ba } @Override - @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_DELETE, eventDescription = "deleting VM backup", async = true) + @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_DELETE, eventDescription = "deleting Instance backup", async = true) public boolean deleteBackup(final Long backupId, final Boolean forced) throws ResourceAllocationException { final BackupVO backup = backupDao.findByIdIncludingRemoved(backupId); if (backup == null) { @@ -1776,12 +1771,12 @@ private boolean attachVolumeToVM(Long zoneId, String restoredVolumeLocation, Bac HypervisorGuru guru = hypervisorGuruManager.getGuru(vm.getHypervisorType()); backupVolumeInfo.setType(Volume.Type.DATADISK); - logger.info("Attaching the restored volume {} to VM {}.", () -> ReflectionToStringBuilder.toString(backupVolumeInfo, ToStringStyle.JSON_STYLE), () -> vm); + logger.info("Attaching the restored Volume {} to Instance {}.", () -> ReflectionToStringBuilder.toString(backupVolumeInfo, ToStringStyle.JSON_STYLE), () -> vm); StoragePoolVO pool = primaryDataStoreDao.findByUuid(datastoreUuid); try { return guru.attachRestoredVolumeToVirtualMachine(zoneId, restoredVolumeLocation, backupVolumeInfo, vm, pool.getId(), backup); } catch (Exception e) { - throw new CloudRuntimeException("Error attach restored volume to VM " + vm.getUuid() + " due to: " + e.getMessage()); + throw new CloudRuntimeException("Error attach restored Volume to Instance " + vm.getUuid() + " due to: " + e.getMessage()); } } @@ -1964,7 +1959,7 @@ private void checkStatusOfCurrentlyExecutingBackups() { case FAILED: final Date nextDateTime = scheduleNextBackupJob(backupSchedule); final String nextScheduledTime = DateUtil.displayDateInTimezone(DateUtil.GMT_TIMEZONE, nextDateTime); - logger.debug("Next backup scheduled time for VM ID " + backupSchedule.getVmId() + " is " + nextScheduledTime); + logger.debug("Next backup scheduled time for Instance ID " + backupSchedule.getVmId() + " is " + nextScheduledTime); break; default: logger.debug("Found async backup job [id: {}, uuid: {}, vmId: {}] with " + @@ -2004,7 +1999,7 @@ public void scheduleBackups() { final Account backupAccount = accountService.getAccount(vm.getAccountId()); if (backupAccount == null || backupAccount.getState() == Account.State.DISABLED) { - logger.debug("Skip backup for VM ({}) since its account has been removed or disabled.", vm); + logger.debug("Skip backup for Instance ({}) since its account has been removed or disabled.", vm); continue; } @@ -2021,7 +2016,7 @@ public void scheduleBackups() { tmpBackupScheduleVO = backupScheduleDao.acquireInLockTable(backupScheduleId); final Long eventId = ActionEventUtils.onScheduledActionEvent(User.UID_SYSTEM, vm.getAccountId(), - EventTypes.EVENT_VM_BACKUP_CREATE, "creating backup for VM ID:" + vm.getUuid(), + EventTypes.EVENT_VM_BACKUP_CREATE, "creating Backup for Instance ID:" + vm.getUuid(), vmId, ApiCommandResourceType.VirtualMachine.toString(), true, 0); final Map params = new HashMap(); @@ -2085,7 +2080,7 @@ protected void runInContext() { private VMInstanceVO findVmById(final Long vmId) { final VMInstanceVO vm = vmInstanceDao.findById(vmId); if (vm == null) { - throw new CloudRuntimeException(String.format("Can't find any VM with ID: [%s].", vmId)); + throw new CloudRuntimeException(String.format("Can't find any Instance with ID: [%s].", vmId)); } return vm; } @@ -2269,7 +2264,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { backup.getBackupOfferingId(), backup.getAccountId(), backup.getDomainId(), backup.getZoneId())); ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_VM_BACKUP_CREATE, - String.format("Created backup %s for VM ID: %s", backup.getUuid(), vm.getUuid()), + String.format("Created Backup %s for Instance ID: %s", backup.getUuid(), vm.getUuid()), vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0); } } diff --git a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java index 8094bddf47fb..27dd630e470f 100644 --- a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java +++ b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java @@ -578,7 +578,7 @@ public void tryRestoreVMTestRestoreFails() throws NoTransitionException { backupManager.tryRestoreVM(backup, vm, offering, "Checking message error."); fail("An exception is needed."); } catch (CloudRuntimeException e) { - assertEquals("Error restoring VM from backup [Checking message error.].", e.getMessage()); + assertEquals("Error restoring Instance from Backup [Checking message error.].", e.getMessage()); } } } @@ -2215,7 +2215,7 @@ public void testRestoreBackupVmNotStopped() { CloudRuntimeException exception = Assert.assertThrows(CloudRuntimeException.class, () -> backupManager.restoreBackup(backupId)); - assertEquals("Existing VM should be stopped before being restored from backup", exception.getMessage()); + assertEquals("Existing Instance should be stopped before being restored from Backup", exception.getMessage()); verify(backupDao, times(1)).findById(backupId); verify(vmInstanceDao, times(1)).findByIdIncludingRemoved(vmId); } @@ -2252,7 +2252,7 @@ public void testRestoreBackupVolumeMismatch() { CloudRuntimeException exception = Assert.assertThrows(CloudRuntimeException.class, () -> backupManager.restoreBackup(backupId)); - assertEquals("Unable to restore VM with the current backup as the backup has different number of disks as the VM", exception.getMessage()); + assertEquals("Unable to restore Instance with the current Backup as the Backup has different number of disks to the Instance", exception.getMessage()); } verify(backupDao, times(1)).findById(backupId); verify(vmInstanceDao, times(1)).findByIdIncludingRemoved(vmId); From 2ae1015073b4351ef855ec90e431676a26977776 Mon Sep 17 00:00:00 2001 From: dahn Date: Tue, 26 May 2026 10:19:19 +0200 Subject: [PATCH 02/28] Add jmsperu to the list of contributors --- .asf.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.asf.yaml b/.asf.yaml index 5105ede1813f..d72395af5704 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -55,6 +55,7 @@ github: - erikbocks - Imvedansh - Damans227 + - jmsperu protected_branches: ~ From 583a1e0dfb04501b2d81413016cfee8d1300f6a2 Mon Sep 17 00:00:00 2001 From: Vishesh <8760112+vishesh92@users.noreply.github.com> Date: Tue, 26 May 2026 16:23:19 +0530 Subject: [PATCH 03/28] Refactor github actions (#12774) --- .github/actions/install-nonoss/action.yml | 31 +++++ .github/actions/setup-env/action.yml | 58 ++++++++++ .github/workflows/build.yml | 40 +++---- .github/workflows/ci.yml | 109 ++++++++++-------- .github/workflows/codecov.yml | 59 ---------- .github/workflows/codeql-analysis.yml | 10 +- .github/workflows/daily-repo-status.lock.yml | 10 +- .../workflows/docker-cloudstack-simulator.yml | 8 +- .github/workflows/main-sonar-check.yml | 61 +++++----- .github/workflows/merge-conflict-checker.yml | 20 ++-- .github/workflows/pre-commit.yml | 18 ++- .github/workflows/rat.yml | 27 ++--- .github/workflows/sonar-check.yml | 60 +++++----- .github/workflows/stale.yml | 4 +- .github/workflows/ui.yml | 12 +- 15 files changed, 275 insertions(+), 252 deletions(-) create mode 100644 .github/actions/install-nonoss/action.yml create mode 100644 .github/actions/setup-env/action.yml delete mode 100644 .github/workflows/codecov.yml diff --git a/.github/actions/install-nonoss/action.yml b/.github/actions/install-nonoss/action.yml new file mode 100644 index 000000000000..39a03213c29d --- /dev/null +++ b/.github/actions/install-nonoss/action.yml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Install CloudStack Non-OSS' +description: 'Clones and installs the shapeblue/cloudstack-nonoss repository.' + +runs: + using: "composite" + steps: + - name: Install cloudstack-nonoss + shell: bash + run: | + git clone --depth 1 https://github.com/shapeblue/cloudstack-nonoss.git nonoss + cd nonoss + bash -x install-non-oss.sh + cd .. + rm -fr nonoss diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml new file mode 100644 index 000000000000..0f8425229242 --- /dev/null +++ b/.github/actions/setup-env/action.yml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Setup CloudStack Environment' +description: 'Sets up JDK (with Maven cache), optionally Python, and optionally APT build dependencies for CloudStack.' + +inputs: + java-version: + description: 'The JDK version to use' + required: false + default: '17' + install-python: + description: 'Whether to install Python 3.10' + required: false + default: 'false' + install-apt-deps: + description: 'Whether to install CloudStack APT build dependencies' + required: false + default: 'false' + +runs: + using: "composite" + steps: + - name: Set up JDK ${{ inputs.java-version }} + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + java-version: ${{ inputs.java-version }} + distribution: 'adopt' + architecture: x64 + cache: 'maven' + + - name: Set up Python + if: ${{ inputs.install-python == 'true' }} + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.10' + architecture: x64 + + - name: Install Build Dependencies + if: ${{ inputs.install-apt-deps == 'true' }} + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y git uuid-runtime genisoimage netcat-openbsd ipmitool build-essential libgcrypt20 libgpg-error-dev libgpg-error0 libopenipmi0 libpython3-dev libssl-dev libffi-dev python3-openssl python3-dev python3-setuptools diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4c33a1313436..4eb508f5f6f8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,40 +16,27 @@ # under the License. name: Build - -on: [push, pull_request] - +on: + - push + - pull_request concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} permissions: contents: read - jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 - - - name: Set up JDK 17 - uses: actions/setup-java@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' + persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v6 + - name: Setup Environment + uses: ./.github/actions/setup-env with: - python-version: '3.10' - architecture: 'x64' - - - name: Install Build Dependencies - run: | - sudo apt-get update - sudo apt-get install -y git uuid-runtime genisoimage netcat ipmitool build-essential libgcrypt20 libgpg-error-dev libgpg-error0 libopenipmi0 ipmitool libpython3-dev libssl-dev libffi-dev python3-openssl python3-dev python3-setuptools - + install-python: 'true' + install-apt-deps: 'true' - name: Env details run: | uname -a @@ -60,9 +47,8 @@ jobs: free -m nproc git status - + - name: Install Non-OSS + uses: ./.github/actions/install-nonoss - name: Noredist Build run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss && cd nonoss && bash -x install-non-oss.sh && cd .. - rm -fr nonoss mvn -B -P developer,systemvm -Dsimulator -Dnoredist clean install -T$(nproc) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52c47b26de8f..e9d03bf1209b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,21 +16,56 @@ # under the License. name: Simulator CI - -on: [push, pull_request] - +on: + - push + - pull_request concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} permissions: contents: read - jobs: build: if: github.repository == 'apache/cloudstack' runs-on: ubuntu-24.04 - + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env + with: + install-python: 'true' + install-apt-deps: 'true' + - name: Env details + run: | + uname -a + whoami + javac -version + mvn -v + python3 --version + free -m + nproc + git status + ipmitool -V + - name: Build with Maven + run: | + mvn -B -P developer,systemvm -Dsimulator clean install -DskipTests=true -T$(nproc) + - name: Archive artifacts + run: | + mkdir -p /tmp/artifacts + tar -czf /tmp/artifacts/targets.tar.gz $(find . -name "target" -type d) tools/marvin/dist engine/schema/dist utils/conf + tar -czf /tmp/artifacts/m2-cloudstack.tar.gz -C ~/.m2/repository org/apache/cloudstack + - name: Upload artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: build-artifacts + path: /tmp/artifacts/ + test: + needs: build + if: github.repository == 'apache/cloudstack' + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -215,30 +250,16 @@ jobs: smoke/test_list_service_offerings smoke/test_list_storage_pools smoke/test_list_volumes"] - steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - - - name: Set up JDK 17 - uses: actions/setup-java@v5 + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.10' - architecture: 'x64' - - - name: Install Build Dependencies - run: | - sudo apt-get update - sudo apt-get install -y git uuid-runtime genisoimage netcat-openbsd ipmitool build-essential libgcrypt20 libgpg-error-dev libgpg-error0 libopenipmi0 ipmitool libpython3-dev libssl-dev libffi-dev python3-openssl python3-dev python3-setuptools - + install-python: 'true' + install-apt-deps: 'true' - name: Setup IPMI Tool for CloudStack run: | # Create cloudstack-common directory if it doesn't exist @@ -256,28 +277,13 @@ jobs: /usr/share/cloudstack-common/ipmitool -C3 $@ EOF sudo chmod 755 /usr/bin/ipmitool - - name: Install Python dependencies run: | python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycryptodome mock flask netaddr pylint pycodestyle six astroid pynose - - name: Install jacoco dependencies run: | wget https://github.com/jacoco/jacoco/releases/download/v0.8.10/jacoco-0.8.10.zip unzip jacoco-0.8.10.zip -d jacoco - - - name: Env details - run: | - uname -a - whoami - javac -version - mvn -v - python3 --version - free -m - nproc - git status - ipmitool -V - - name: Setup MySQL Server run: | # https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2004-Readme.md#mysql @@ -286,25 +292,28 @@ jobs: sudo mysql -uroot -proot -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH caching_sha2_password BY ''; FLUSH PRIVILEGES;" sudo systemctl restart mysql sudo mysql -uroot -e "SELECT VERSION();" - - - name: Build with Maven + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: build-artifacts + path: /tmp/artifacts/ + - name: Extract artifacts run: | - mvn -B -P developer,systemvm -Dsimulator clean install -DskipTests=true -T$(nproc) - + tar -xzf /tmp/artifacts/targets.tar.gz + mkdir -p ~/.m2/repository + tar -xzf /tmp/artifacts/m2-cloudstack.tar.gz -C ~/.m2/repository - name: Setup Simulator Prerequisites run: | sudo python3 -m pip install --upgrade netaddr mysql-connector-python python3 -m pip install --user --upgrade tools/marvin/dist/[mM]arvin-*.tar.gz mvn -q -Pdeveloper -pl developer -Ddeploydb mvn -q -Pdeveloper -pl developer -Ddeploydb-simulator - - name: Generate jacoco-coverage.sh run: | echo "java -jar jacoco/lib/jacococli.jar report jacoco-it.exec \\" > jacoco-report.sh find . | grep "target/classes" | sed 's/\/classes\//\/classes /g' | awk '{print "--classfiles", $1, "\\"}' | sort |uniq >> jacoco-report.sh find . | grep "src/main/java" | sed 's/\/java\//\/java /g' | awk '{print "--sourcefiles", $1, "\\"}' | sort | uniq >> jacoco-report.sh echo "--xml jacoco-coverage.xml" >> jacoco-report.sh - - name: Start CloudStack Management Server with Simulator run: | export MAVEN_OPTS="-Xmx4096m -XX:MaxMetaspaceSize=800m -Djava.security.egd=file:/dev/urandom -javaagent:jacoco/lib/jacocoagent.jar=address=*,port=36320,output=tcpserver --add-opens=java.base/java.lang=ALL-UNNAMED --add-exports=java.base/sun.security.x509=ALL-UNNAMED --add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED" @@ -315,7 +324,6 @@ jobs: set -e echo -e "\nStarting Advanced Zone DataCenter deployment" python3 tools/marvin/marvin/deployDataCenter.py -i setup/dev/advdualzone.cfg 2>&1 || true - - name: Run Integration Tests with Simulator run: | mkdir -p integration-test-results/smoke/misc @@ -335,13 +343,12 @@ jobs: bash jacoco-report.sh mvn -Dsimulator -pl client jetty:stop 2>&1 find /tmp//MarvinLogs -type f -exec echo -e "Printing marvin logs {} :\n" \; -exec cat {} \; - - name: Integration Tests Result run: | echo -e "Simulator CI Test Results: (only failures listed)\n" python3 ./tools/marvin/xunit-reader.py integration-test-results/ - - uses: codecov/codecov-action@v6 + - uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: files: jacoco-coverage.xml fail_ci_if_error: true diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml deleted file mode 100644 index 0ee10baa385b..000000000000 --- a/.github/workflows/codecov.yml +++ /dev/null @@ -1,59 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Coverage Check - -on: [pull_request, push] - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - build: - if: github.repository == 'apache/cloudstack' - name: codecov - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Set up JDK 17 - uses: actions/setup-java@v5 - with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - - - name: Build CloudStack with Quality Checks - run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss - cd nonoss && bash -x install-non-oss.sh && cd .. - mvn -P quality -Dsimulator -Dnoredist clean install -T$(nproc) - - - uses: codecov/codecov-action@v6 - with: - files: ./client/target/site/jacoco-aggregate/jacoco.xml - fail_ci_if_error: true - flags: unittests - verbose: true - name: codecov - token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index cb1fa88a1023..a2d581dcac20 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -35,14 +35,16 @@ jobs: language: ["actions"] steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@v4 + uses: github/codeql-action/init@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@v4 + uses: github/codeql-action/autobuild@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 + uses: github/codeql-action/analyze@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 with: category: "Security" diff --git a/.github/workflows/daily-repo-status.lock.yml b/.github/workflows/daily-repo-status.lock.yml index 35eb5d409a48..f8ce13c3cee8 100644 --- a/.github/workflows/daily-repo-status.lock.yml +++ b/.github/workflows/daily-repo-status.lock.yml @@ -102,7 +102,7 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0. with: - persist-credentials: + persist-credentials: false - name: Create gh-aw temp run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir. - name: Configure Git @@ -789,7 +789,7 @@ jobs: destination: /opt/gh-aw/ - name: Download agent output continue-on-error: - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v6.0. + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: agent- path: /tmp/gh-aw/safeoutputs/ @@ -886,13 +886,13 @@ jobs: destination: /opt/gh-aw/ - name: Download agent continue-on-error: - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v6.0. + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: agent- path: /tmp/gh-aw/threat-detection/ - name: Download agent output continue-on-error: - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v6.0. + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: agent- path: /tmp/gh-aw/threat-detection/ @@ -998,7 +998,7 @@ jobs: destination: /opt/gh-aw/ - name: Download agent output continue-on-error: - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v6.0. + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: agent- path: /tmp/gh-aw/safeoutputs/ diff --git a/.github/workflows/docker-cloudstack-simulator.yml b/.github/workflows/docker-cloudstack-simulator.yml index 96c9400935c2..dc00968563d0 100644 --- a/.github/workflows/docker-cloudstack-simulator.yml +++ b/.github/workflows/docker-cloudstack-simulator.yml @@ -35,10 +35,10 @@ concurrency: jobs: build: if: github.repository == 'apache/cloudstack' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Login to Docker Registry - uses: docker/login-action@v4 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKERHUB_USER }} @@ -47,7 +47,9 @@ jobs: - name: Set Docker repository name run: echo "DOCKER_REPOSITORY=apache" >> $GITHUB_ENV - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Set ACS version run: echo "ACS_VERSION=$(grep '' pom.xml | head -2 | tail -1 | cut -d'>' -f2 |cut -d'<' -f1)" >> $GITHUB_ENV diff --git a/.github/workflows/main-sonar-check.yml b/.github/workflows/main-sonar-check.yml index 7ccd6600ab97..ad58d07a9894 100644 --- a/.github/workflows/main-sonar-check.yml +++ b/.github/workflows/main-sonar-check.yml @@ -15,54 +15,51 @@ # specific language governing permissions and limitations # under the License. -name: Main Branch Sonar Quality Check - +name: Sonar Quality Check (Main) +permissions: + contents: read on: push: branches: - main - -permissions: - contents: read # to fetch code (actions/checkout) - pull-requests: write # for sonar to comment on pull-request - +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: build: if: github.repository == 'apache/cloudstack' - name: Main Sonar JaCoCo Build - runs-on: ubuntu-22.04 + name: Sonar JaCoCo Coverage + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - - - name: Set up JDK17 - uses: actions/setup-java@v5 + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - + install-python: 'true' + install-apt-deps: 'true' - name: Cache SonarCloud packages - uses: actions/cache@v5 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar - - - name: Cache local Maven repository - uses: actions/cache@v5 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2 - - - name: Run Tests with Coverage + - name: Install Non-OSS + uses: ./.github/actions/install-nonoss + - name: Run Build and Tests with Coverage + run: mvn -B -T$(nproc) -P developer,systemvm,quality -Dsimulator -Dnoredist clean install + - name: Upload to SonarQube env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss - cd nonoss && bash -x install-non-oss.sh && cd .. - mvn -T$(nproc) -P quality -Dsimulator -Dnoredist clean install org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack + run: mvn -B -P quality org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack -Dsonar.branch.name=${{ github.ref_name }} + - uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 + with: + files: ./client/target/site/jacoco-aggregate/jacoco.xml + fail_ci_if_error: true + flags: unittests + verbose: true + name: codecov + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/merge-conflict-checker.yml b/.github/workflows/merge-conflict-checker.yml index 2c826a47c7ec..f23719f7183a 100644 --- a/.github/workflows/merge-conflict-checker.yml +++ b/.github/workflows/merge-conflict-checker.yml @@ -29,14 +29,14 @@ concurrency: jobs: triage: permissions: - pull-requests: write # for eps1lon/actions-label-merge-conflict to label PRs - runs-on: ubuntu-22.04 + pull-requests: write # for eps1lon/actions-label-merge-conflict to label PRs + runs-on: ubuntu-24.04 steps: - - name: Conflict Check - uses: eps1lon/actions-label-merge-conflict@v3.0.3 - with: - repoToken: "${{ secrets.GITHUB_TOKEN }}" - dirtyLabel: "status:has-conflicts" - removeOnDirtyLabel: "status:ready-for-review" - continueOnMissingPermissions: true - commentOnDirty: "This pull request has merge conflicts. Dear author, please fix the conflicts and sync your branch with the base branch." + - name: Conflict Check + uses: eps1lon/actions-label-merge-conflict@1df065ebe6e3310545d4f4c4e862e43bdca146f0 # v3.0.3 + with: + repoToken: "${{ secrets.GITHUB_TOKEN }}" + dirtyLabel: "status:has-conflicts" + removeOnDirtyLabel: "status:ready-for-review" + continueOnMissingPermissions: true + commentOnDirty: "This pull request has merge conflicts. Dear author, please fix the conflicts and sync your branch with the base branch." diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 895a597659de..1ea00be9112f 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -29,17 +29,23 @@ concurrency: jobs: pre-commit: name: Run pre-commit - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Check Out - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + cache: 'pip' - name: Install - run: | - python -m pip install --upgrade pip - pip install pre-commit + run: pip install pre-commit - name: Set PY run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV - - uses: actions/cache@v5 + - name: Cache pre-commit environments + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.cache/pre-commit key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} diff --git a/.github/workflows/rat.yml b/.github/workflows/rat.yml index 21b8e197d825..a36a956856e7 100644 --- a/.github/workflows/rat.yml +++ b/.github/workflows/rat.yml @@ -16,32 +16,27 @@ # under the License. name: License Check - -on: [push, pull_request] - +on: + - push + - pull_request concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} permissions: contents: read - jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 - - name: Set up JDK 17 - uses: actions/setup-java@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - java-version: '17' - distribution: 'adopt' - architecture: x64 - cache: maven + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env + - name: Install Non-OSS + uses: ./.github/actions/install-nonoss - name: RAT licence checks run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss && cd nonoss && bash -x install-non-oss.sh && cd .. - rm -fr nonoss mvn -P developer,systemvm -Dsimulator -Dnoredist -pl . org.apache.rat:apache-rat-plugin:0.12:check - name: Rat Report if: always() diff --git a/.github/workflows/sonar-check.yml b/.github/workflows/sonar-check.yml index 9f5c3a194bc7..49a49d009580 100644 --- a/.github/workflows/sonar-check.yml +++ b/.github/workflows/sonar-check.yml @@ -16,58 +16,52 @@ # under the License. name: Sonar Quality Check - -on: [pull_request] - permissions: - contents: read # to fetch code (actions/checkout) - pull-requests: write # for sonar to comment on pull-request - + contents: read + pull-requests: write +on: + pull_request: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: build: - if: github.repository == 'apache/cloudstack' && github.event.pull_request.head.repo.full_name == github.repository name: Sonar JaCoCo Coverage - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - ref: "refs/pull/${{ github.event.number }}/merge" fetch-depth: 0 - - - name: Set up JDK17 - uses: actions/setup-java@v5 + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - + install-python: 'true' + install-apt-deps: 'true' - name: Cache SonarCloud packages - uses: actions/cache@v5 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar - - - name: Cache local Maven repository - uses: actions/cache@v5 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2 - + - name: Install Non-OSS + uses: ./.github/actions/install-nonoss - name: Run Build and Tests with Coverage - id: coverage + run: mvn -B -T$(nproc) -P developer,systemvm,quality -Dsimulator -Dnoredist clean install + - name: Upload to SonarQube + if: github.repository == 'apache/cloudstack' && github.event.pull_request.head.repo.full_name == github.repository env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} PR_ID: ${{ github.event.pull_request.number }} HEADREF: ${{ github.event.pull_request.head.ref }} run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss - cd nonoss && bash -x install-non-oss.sh && cd .. - mvn -T$(nproc) -P quality -Dsimulator -Dnoredist clean install org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack -Dsonar.pullrequest.key="$PR_ID" -Dsonar.pullrequest.branch="$HEADREF" -Dsonar.pullrequest.github.repository=apache/cloudstack -Dsonar.pullrequest.provider=GitHub -Dsonar.pullrequest.github.summary_comment=true + mvn -B -P quality org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack -Dsonar.pullrequest.key="$PR_ID" -Dsonar.pullrequest.branch="$HEADREF" -Dsonar.pullrequest.github.repository=apache/cloudstack -Dsonar.pullrequest.provider=GitHub -Dsonar.pullrequest.github.summary_comment=true + - uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 + with: + files: ./client/target/site/jacoco-aggregate/jacoco.xml + fail_ci_if_error: true + flags: unittests + verbose: true + name: codecov + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 842e4497a4ad..5c92f23888e2 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -28,7 +28,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@v10 + - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 with: stale-issue-message: 'This issue is stale because it has been open for 120 days with no activity. It may be removed by administrators of this project at any time. Remove the stale label or comment to request for removal of it to prevent this.' stale-pr-message: 'This PR is stale because it has been open for 120 days with no activity. It may be removed by administrators of this project at any time. Remove the stale label or comment to request for removal of it to prevent this.' @@ -41,7 +41,7 @@ jobs: days-before-pr-close: 240 exempt-issue-labels: 'gsoc,good-first-issue,long-term-plan' exempt-pr-labels: 'status:ready-for-merge,status:needs-testing,status:on-hold' - - uses: actions/stale@v10 + - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 with: stale-issue-label: 'archive' days-before-stale: 240 diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index 2db8456fcba7..ff27809db8f6 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -28,15 +28,19 @@ permissions: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Set up Node - uses: actions/setup-node@v6 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 16 + cache: 'npm' + cache-dependency-path: 'ui/package-lock.json' - name: Env details run: | @@ -55,7 +59,7 @@ jobs: npm run lint npm run test:unit - - uses: codecov/codecov-action@v6 + - uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 if: github.repository == 'apache/cloudstack' with: working-directory: ui From b16340399baed5faec14bf01b6fb99f40ce63c58 Mon Sep 17 00:00:00 2001 From: dahn Date: Tue, 26 May 2026 14:59:07 +0200 Subject: [PATCH 04/28] fix: show re-enable 2FA button when is2faenabled is null/undefined After disabling 2FA via setupUserTwoFactorAuthentication, the API may return is2faenabled as null or undefined rather than boolean false. The strict equality check (=== false) prevented the "Setup 2FA" button from appearing in those cases, making it impossible to re-enable 2FA. Change the check to a falsy check (!record.is2faenabled) so the button is shown whenever 2FA is not enabled, regardless of whether the value is false, null, or undefined. Fixes: https://github.com/apache/cloudstack/issues/13233 --- ui/src/config/section/user.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/config/section/user.js b/ui/src/config/section/user.js index a298033bd963..eaaca983dc0a 100644 --- a/ui/src/config/section/user.js +++ b/ui/src/config/section/user.js @@ -179,7 +179,7 @@ export default { dataView: true, popup: true, show: (record, store) => { - return (record.is2faenabled === false && record.id === store.userInfo.id) + return (!record.is2faenabled && record.id === store.userInfo.id) }, component: shallowRef(defineAsyncComponent(() => import('@/views/iam/SetupTwoFaAtUserProfile.vue'))) }, From 7f849e9362f3f8ef521bd1e97d4786d773f8f4f5 Mon Sep 17 00:00:00 2001 From: GaOrtiga <49285692+GaOrtiga@users.noreply.github.com> Date: Tue, 26 May 2026 10:08:24 -0300 Subject: [PATCH 05/28] add GaOrtiga to colaborators (#13246) --- .asf.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.asf.yaml b/.asf.yaml index d72395af5704..3b2f7691d136 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -56,6 +56,7 @@ github: - Imvedansh - Damans227 - jmsperu + - GaOrtiga protected_branches: ~ From 3f6866d70a6e4eb8a12dd257d33bde5aae79aacf Mon Sep 17 00:00:00 2001 From: Fabricio Duarte Date: Tue, 26 May 2026 10:36:04 -0300 Subject: [PATCH 06/28] Refactor Quota balance (#12961) --- .../apache/cloudstack/api/ApiConstants.java | 3 + .../cloudstack/quota/QuotaManagerImpl.java | 9 +- .../cloudstack/quota/dao/QuotaBalanceDao.java | 10 +- .../quota/dao/QuotaBalanceDaoImpl.java | 187 +++++++----------- .../quota/dao/QuotaBalanceDaoImplTest.java | 91 +++++++++ .../api/command/QuotaBalanceCmd.java | 62 +++--- .../api/response/QuotaBalanceResponse.java | 132 +++---------- .../api/response/QuotaResponseBuilder.java | 7 +- .../response/QuotaResponseBuilderImpl.java | 128 ++---------- .../apache/cloudstack/quota/QuotaService.java | 2 +- .../cloudstack/quota/QuotaServiceImpl.java | 100 +++++----- .../api/command/QuotaBalanceCmdTest.java | 42 ++-- .../QuotaResponseBuilderImplTest.java | 70 ++++--- .../quota/QuotaServiceImplTest.java | 95 ++++++--- .../com/cloud/user/AccountManagerImpl.java | 8 +- .../plugins/quota/test_quota_balance.py | 49 +++-- 16 files changed, 442 insertions(+), 553 deletions(-) create mode 100644 framework/quota/src/test/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImplTest.java diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 4d4ead277e5d..694830ea2f36 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -69,6 +69,8 @@ public class ApiConstants { public static final String BACKUP_VM_OFFERING_REMOVED = "vmbackupofferingremoved"; public static final String IS_BACKUP_VM_EXPUNGED = "isbackupvmexpunged"; public static final String BACKUP_TOTAL = "backuptotal"; + public static final String BALANCE = "balance"; + public static final String BALANCES = "balances"; public static final String BASE64_IMAGE = "base64image"; public static final String BGP_PEERS = "bgppeers"; public static final String BGP_PEER_IDS = "bgppeerids"; @@ -171,6 +173,7 @@ public class ApiConstants { public static final String DATACENTER_NAME = "datacentername"; public static final String DATADISKS_DETAILS = "datadisksdetails"; public static final String DATADISK_OFFERING_LIST = "datadiskofferinglist"; + public static final String DATE = "date"; public static final String DEFAULT_VALUE = "defaultvalue"; public static final String DELETE_PROTECTION = "deleteprotection"; public static final String DESCRIPTION = "description"; diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java index 816144aa2f16..58f51eae9fcd 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java @@ -150,8 +150,9 @@ protected void processQuotaBalanceForAccount(AccountVO accountVo, List> periods = accountQuotaUsages.stream() @@ -215,7 +216,7 @@ protected BigDecimal retrieveBalanceForUsageCalculation(long accountId, long dom logger.debug(String.format("Persisting the first quota balance [%s] for account [%s].", firstBalance, accountToString)); _quotaBalanceDao.saveQuotaBalance(firstBalance); } else { - QuotaBalanceVO lastRealBalance = _quotaBalanceDao.findLastBalanceEntry(accountId, domainId, startDate); + QuotaBalanceVO lastRealBalance = _quotaBalanceDao.getLastQuotaBalanceEntry(accountId, domainId, startDate); if (lastRealBalance == null) { logger.warn("Account [{}] has quota usage entries, however it does not have a quota balance.", accountToString); @@ -244,7 +245,7 @@ protected void saveQuotaAccount(long accountId, BigDecimal aggregatedUsage, Date } protected BigDecimal aggregateCreditBetweenDates(Long accountId, Long domainId, Date startDate, Date endDate, String accountToString) { - List creditsReceived = _quotaBalanceDao.findCreditBalance(accountId, domainId, startDate, endDate); + List creditsReceived = _quotaBalanceDao.findCreditBalances(accountId, domainId, startDate, endDate); logger.debug("Account [{}] has [{}] credit entries before [{}].", accountToString, creditsReceived.size(), DateUtil.displayDateInTimezone(usageAggregationTimeZone, endDate)); diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDao.java index c694eaeefbe8..2964746bdf06 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDao.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDao.java @@ -28,16 +28,14 @@ public interface QuotaBalanceDao extends GenericDao { QuotaBalanceVO saveQuotaBalance(QuotaBalanceVO qb); - List findCreditBalance(Long accountId, Long domainId, Date startDate, Date endDate); + List findCreditBalances(Long accountId, Long domainId, Date startDate, Date endDate); - QuotaBalanceVO findLastBalanceEntry(Long accountId, Long domainId, Date beforeThis); + QuotaBalanceVO getLastQuotaBalanceEntry(Long accountId, Long domainId, Date beforeThis); QuotaBalanceVO findLaterBalanceEntry(Long accountId, Long domainId, Date afterThis); - List findQuotaBalance(Long accountId, Long domainId, Date startDate, Date endDate); + List listQuotaBalances(Long accountId, Long domainId, Date startDate, Date endDate); - List lastQuotaBalanceVO(Long accountId, Long domainId, Date startDate); - - BigDecimal lastQuotaBalance(Long accountId, Long domainId, Date startDate); + BigDecimal getLastQuotaBalance(Long accountId, Long domainId); } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImpl.java index 01272d1a6184..21553ebd27b0 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImpl.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImpl.java @@ -18,11 +18,14 @@ import java.math.BigDecimal; import java.util.ArrayList; -import java.util.Collections; import java.util.Date; import java.util.List; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.time.DateUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; import com.cloud.utils.db.Filter; @@ -32,160 +35,104 @@ import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; import com.cloud.utils.db.TransactionLegacy; -import com.cloud.utils.db.TransactionStatus; @Component public class QuotaBalanceDaoImpl extends GenericDaoBase implements QuotaBalanceDao { + private static final Logger logger = LogManager.getLogger(QuotaBalanceDaoImpl.class); @Override - public QuotaBalanceVO findLastBalanceEntry(final Long accountId, final Long domainId, final Date beforeThis) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback() { - @Override - public QuotaBalanceVO doInTransaction(final TransactionStatus status) { - List quotaBalanceEntries = new ArrayList<>(); - Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", false, 0L, 1L); - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); + public QuotaBalanceVO getLastQuotaBalanceEntry(final Long accountId, final Long domainId, final Date beforeThis) { + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback) status -> { + Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", false, 0L, 1L); + QueryBuilder qb = getQuotaBalanceQueryBuilder(accountId, domainId); + qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); + + if (beforeThis != null) { qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.LT, beforeThis); - quotaBalanceEntries = search(qb.create(), filter); - return !quotaBalanceEntries.isEmpty() ? quotaBalanceEntries.get(0) : null; } + + List quotaBalanceEntries = search(qb.create(), filter); + return !quotaBalanceEntries.isEmpty() ? quotaBalanceEntries.get(0) : null; }); } @Override public QuotaBalanceVO findLaterBalanceEntry(final Long accountId, final Long domainId, final Date afterThis) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback() { - @Override - public QuotaBalanceVO doInTransaction(final TransactionStatus status) { - List quotaBalanceEntries = new ArrayList<>(); - Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true, 0L, 1L); - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); - qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.GT, afterThis); - quotaBalanceEntries = search(qb.create(), filter); - return quotaBalanceEntries.size() > 0 ? quotaBalanceEntries.get(0) : null; - } + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback) status -> { + Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true, 0L, 1L); + QueryBuilder qb = getQuotaBalanceQueryBuilder(accountId, domainId); + qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); + qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.GT, afterThis); + + List quotaBalanceEntries = search(qb.create(), filter); + return !quotaBalanceEntries.isEmpty() ? quotaBalanceEntries.get(0) : null; }); } @Override public QuotaBalanceVO saveQuotaBalance(final QuotaBalanceVO qb) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback() { - @Override - public QuotaBalanceVO doInTransaction(final TransactionStatus status) { - return persist(qb); - } - }); + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback) status -> persist(qb)); } @Override - public List findCreditBalance(final Long accountId, final Long domainId, final Date lastbalancedate, final Date beforeThis) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback>() { - @Override - public List doInTransaction(final TransactionStatus status) { - if ((lastbalancedate != null) && (beforeThis != null) && lastbalancedate.before(beforeThis)) { - Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true, 0L, Long.MAX_VALUE); - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.GT, 0); - qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.BETWEEN, lastbalancedate, beforeThis); - return search(qb.create(), filter); - } else { - return new ArrayList(); - } + public List findCreditBalances(final Long accountId, final Long domainId, final Date startDate, final Date endDate) { + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback>) status -> { + if (ObjectUtils.anyNull(startDate, endDate) || startDate.after(endDate)) { + return new ArrayList<>(); } - }); - } - @Override - public List findQuotaBalance(final Long accountId, final Long domainId, final Date startDate, final Date endDate) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback>() { - @Override - public List doInTransaction(final TransactionStatus status) { - List quotaUsageRecords = null; - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - if (accountId != null) { - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - } - if (domainId != null) { - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - } - if ((startDate != null) && (endDate != null) && startDate.before(endDate)) { - qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.BETWEEN, startDate, endDate); - } else { - return Collections. emptyList(); - } - quotaUsageRecords = listBy(qb.create()); - if (quotaUsageRecords.size() == 0) { - quotaUsageRecords.addAll(lastQuotaBalanceVO(accountId, domainId, startDate)); - } - return quotaUsageRecords; + Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true, 0L, Long.MAX_VALUE); + QueryBuilder qb = getQuotaBalanceQueryBuilder(accountId, domainId); + qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.GT, 0); + qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.BETWEEN, startDate, endDate); - } + return search(qb.create(), filter); }); - } @Override - public List lastQuotaBalanceVO(final Long accountId, final Long domainId, final Date pivotDate) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback>() { - @Override - public List doInTransaction(final TransactionStatus status) { - List quotaUsageRecords = null; - List trimmedRecords = new ArrayList(); - Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", false, 0L, 100L); - // ASSUMPTION there will be less than 100 continuous credit - // transactions - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - if (accountId != null) { - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - } - if (domainId != null) { - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - } - if ((pivotDate != null)) { - qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.LTEQ, pivotDate); - } - quotaUsageRecords = search(qb.create(), filter); - - // get records before startDate to find start balance - for (QuotaBalanceVO entry : quotaUsageRecords) { - if (logger.isDebugEnabled()) { - logger.debug("FindQuotaBalance Entry=" + entry); - } - if (entry.getCreditsId() > 0) { - trimmedRecords.add(entry); - } else { - trimmedRecords.add(entry); - break; // add only consecutive credit entries and last balance entry - } - } - return trimmedRecords; + public List listQuotaBalances(final Long accountId, final Long domainId, final Date startDate, final Date endDate) { + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback>) status -> { + QueryBuilder qb = getQuotaBalanceQueryBuilder(accountId, domainId); + qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); + if (startDate != null) { + qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.GTEQ, startDate); + } + if (endDate != null) { + qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.LTEQ, endDate); } + + Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true); + return listBy(qb.create(), filter); }); } @Override - public BigDecimal lastQuotaBalance(final Long accountId, final Long domainId, Date startDate) { - List quotaBalance = lastQuotaBalanceVO(accountId, domainId, startDate); - BigDecimal finalBalance = new BigDecimal(0); - if (quotaBalance.isEmpty()) { - logger.info("There are no balance entries on or before the requested date."); - return finalBalance; + public BigDecimal getLastQuotaBalance(final Long accountId, final Long domainId) { + QuotaBalanceVO quotaBalance = getLastQuotaBalanceEntry(accountId, domainId, null); + BigDecimal finalBalance = BigDecimal.ZERO; + Date startDate = DateUtils.addDays(new Date(), -1); + if (quotaBalance == null) { + logger.info("There are no balance entries for account [{}] and domain [{}]. Considering only new added credits.", accountId, domainId); + } else { + finalBalance = quotaBalance.getCreditBalance(); + startDate = quotaBalance.getUpdatedOn(); } - for (QuotaBalanceVO entry : quotaBalance) { - if (logger.isDebugEnabled()) { - logger.debug("lastQuotaBalance Entry=" + entry); - } - finalBalance = finalBalance.add(entry.getCreditBalance()); + + List credits = findCreditBalances(accountId, domainId, startDate, new Date()); + + for (QuotaBalanceVO credit : credits) { + finalBalance = finalBalance.add(credit.getCreditBalance()); } + return finalBalance; } + private QueryBuilder getQuotaBalanceQueryBuilder(Long accountId, Long domainId) { + QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); + qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); + qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); + return qb; + } + } diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImplTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImplTest.java new file mode 100644 index 000000000000..afdd88f8d1da --- /dev/null +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImplTest.java @@ -0,0 +1,91 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.quota.dao; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Arrays; + +import org.apache.cloudstack.quota.vo.QuotaBalanceVO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class QuotaBalanceDaoImplTest { + QuotaBalanceDaoImpl quotaBalanceDaoImplSpy = Mockito.spy(QuotaBalanceDaoImpl.class); + + @Mock + QuotaBalanceVO quotaBalanceVoMock; + + @Test + public void getLastQuotaBalanceTestLastEntryIsNullAndNoCreditsReturnsZero() { + Mockito.doReturn(null).when(quotaBalanceDaoImplSpy).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(new ArrayList<>()).when(quotaBalanceDaoImplSpy).findCreditBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + + BigDecimal result = quotaBalanceDaoImplSpy.getLastQuotaBalance(1L, 2L); + + Assert.assertEquals(BigDecimal.ZERO, result); + } + + @Test + public void getLastQuotaBalanceTestReturnsLastEntryAndNoCredits() { + BigDecimal expected = BigDecimal.valueOf(-1542.46); + Mockito.doReturn(quotaBalanceVoMock).when(quotaBalanceDaoImplSpy).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(expected).when(quotaBalanceVoMock).getCreditBalance(); + Mockito.doReturn(new ArrayList<>()).when(quotaBalanceDaoImplSpy).findCreditBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + + BigDecimal result = quotaBalanceDaoImplSpy.getLastQuotaBalance(5L, 8L); + + Assert.assertEquals(expected, result); + } + + @Test + public void getLastQuotaBalanceTestReturnsLastEntryPlusCredits() { + BigDecimal balance = BigDecimal.valueOf(-1542.46); + BigDecimal credit1 = new BigDecimal("150.14"); + BigDecimal credit2 = new BigDecimal("78.96"); + BigDecimal expected = balance.add(credit1).add(credit2); + + Mockito.doReturn(quotaBalanceVoMock).when(quotaBalanceDaoImplSpy).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(balance, credit1, credit2).when(quotaBalanceVoMock).getCreditBalance(); + Mockito.doReturn(Arrays.asList(quotaBalanceVoMock, quotaBalanceVoMock)).when(quotaBalanceDaoImplSpy).findCreditBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + + BigDecimal result = quotaBalanceDaoImplSpy.getLastQuotaBalance(5L, 8L); + + Assert.assertEquals(expected, result); + } + + @Test + public void getLastQuotaBalanceTestReturnsLastEntryIsNullPlusCredits() { + BigDecimal credit1 = new BigDecimal("150.14"); + BigDecimal credit2 = new BigDecimal("78.96"); + BigDecimal expected = credit1.add(credit2); + + Mockito.doReturn(null).when(quotaBalanceDaoImplSpy).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(credit1, credit2).when(quotaBalanceVoMock).getCreditBalance(); + Mockito.doReturn(Arrays.asList(quotaBalanceVoMock, quotaBalanceVoMock)).when(quotaBalanceDaoImplSpy).findCreditBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + + BigDecimal result = quotaBalanceDaoImplSpy.getLastQuotaBalance(5L, 8L); + + Assert.assertEquals(expected, result); + } +} diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java index 0cec0df66182..e5c89c5de1c5 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java @@ -17,12 +17,9 @@ package org.apache.cloudstack.api.command; import java.util.Date; -import java.util.List; import javax.inject.Inject; -import com.cloud.user.Account; - import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; @@ -30,36 +27,41 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.QuotaBalanceResponse; import org.apache.cloudstack.api.response.QuotaResponseBuilder; -import org.apache.cloudstack.quota.vo.QuotaBalanceVO; -import org.apache.cloudstack.api.response.QuotaStatementItemResponse; +import org.apache.commons.lang3.ObjectUtils; -@APICommand(name = "quotaBalance", responseObject = QuotaStatementItemResponse.class, description = "Create a quota balance statement", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, +@APICommand(name = "quotaBalance", responseObject = QuotaBalanceResponse.class, description = "Create Quota balance statements for the provided Accounts or Projects.", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, httpMethod = "GET") public class QuotaBalanceCmd extends BaseCmd { - - - @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = true, description = "Account Id for which statement needs to be generated") + @ACL + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "Name of the Account for which balance statements will be generated. " + + "Deprecated, please use 'accountid' instead.") private String accountName; @ACL - @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "If domain Id is given and the caller is domain admin then the statement is generated for domain.") + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "ID of the domain which the Account identified by 'account' belongs to." + + "Deprecated, please use 'accountid' instead.") private Long domainId; - @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "End of the period of the Quota balance." + - ApiConstants.PARAMETER_DESCRIPTION_END_DATE_POSSIBLE_FORMATS) + @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "Date of the last Quota balance to be returned. Must be informed together with the " + + "startdate parameter, and can not be before startdate. " + ApiConstants.PARAMETER_DESCRIPTION_END_DATE_POSSIBLE_FORMATS) private Date endDate; - @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "Start of the period of the Quota balance. " + + @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "Date of the first Quota balance to be returned. Must be before today. " + ApiConstants.PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS) private Date startDate; @ACL - @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "List usage records for the specified Account") + @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "ID of the Account for which balance statements will be generated.") private Long accountId; + @ACL + @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "ID of the Project for which balance statements will be generated. Can not be specified with accountid.") + private Long projectId; + @Inject QuotaResponseBuilder _responseBuilder; @@ -88,48 +90,32 @@ public void setDomainId(Long domainId) { } public Date getEndDate() { - if (endDate == null){ - return null; - } - else{ - return _responseBuilder.startOfNextDay(new Date(endDate.getTime())); - } + return endDate; } public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); + this.endDate = endDate; } public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + return startDate; } public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); + this.startDate = startDate; } @Override public long getEntityOwnerId() { - if (accountId != null) { - return accountId; - } - Account account = _accountService.getActiveAccountByName(accountName, domainId); - if (account != null) { - return account.getAccountId(); + if (ObjectUtils.allNull(accountId, accountName, domainId, projectId)) { + return -1; } - return Account.ACCOUNT_ID_SYSTEM; + return _accountService.finalizeAccountId(accountId, accountName, domainId, projectId); } @Override public void execute() { - List quotaUsage = _responseBuilder.getQuotaBalance(this); - - QuotaBalanceResponse response; - if (endDate == null) { - response = _responseBuilder.createQuotaLastBalanceResponse(quotaUsage, getStartDate()); - } else { - response = _responseBuilder.createQuotaBalanceResponse(quotaUsage, getStartDate(), new Date(endDate.getTime())); - } + QuotaBalanceResponse response = _responseBuilder.createQuotaBalanceResponse(this); response.setResponseName(getCommandName()); setResponseObject(response); } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaBalanceResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaBalanceResponse.java index 625f4829c673..714066b048ad 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaBalanceResponse.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaBalanceResponse.java @@ -17,137 +17,65 @@ package org.apache.cloudstack.api.response; import java.math.BigDecimal; -import java.math.RoundingMode; -import java.util.ArrayList; import java.util.Date; import java.util.List; import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; -import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import com.cloud.serializer.Param; public class QuotaBalanceResponse extends BaseResponse { - @SerializedName("accountid") - @Param(description = "Account ID") - private Long accountId; - - @SerializedName("account") - @Param(description = "Account name") - private String accountName; - - @SerializedName("domain") - @Param(description = "Domain ID") - private Long domainId; - - @SerializedName("startquota") - @Param(description = "Quota started with") - private BigDecimal startQuota; - - @SerializedName("endquota") - @Param(description = "Quota by end of this period") - private BigDecimal endQuota; - - @SerializedName("credits") - @Param(description = "List of credits made during this period") - private List credits = null; - - @SerializedName("startdate") - @Param(description = "Start date") - private Date startDate = null; - - @SerializedName("enddate") - @Param(description = "End date") - private Date endDate = null; - - @SerializedName("currency") - @Param(description = "Currency") + @SerializedName(ApiConstants.CURRENCY) + @Param(description = "Balance's currency.") private String currency; - public QuotaBalanceResponse() { - super(); - credits = new ArrayList(); - } - - public Long getAccountId() { - return accountId; - } - - public void setAccountId(Long accountId) { - this.accountId = accountId; - } - - public String getAccountName() { - return accountName; - } + @SerializedName(ApiConstants.DATE) + @Param(description = "Balance's date.") + private Date date; - public void setAccountName(String accountName) { - this.accountName = accountName; - } + @SerializedName(ApiConstants.BALANCE) + @Param(description = "Balance's value.") + private BigDecimal balance; - public Long getDomainId() { - return domainId; - } - - public void setDomainId(Long domainId) { - this.domainId = domainId; - } + @SerializedName(ApiConstants.BALANCES) + @Param(description = "List of balances in the period.") + private List balances; - public BigDecimal getStartQuota() { - return startQuota; - } - - public void setStartQuota(BigDecimal startQuota) { - this.startQuota = startQuota.setScale(2, RoundingMode.HALF_EVEN); - } - - public BigDecimal getEndQuota() { - return endQuota; - } - - public void setEndQuota(BigDecimal endQuota) { - this.endQuota = endQuota.setScale(2, RoundingMode.HALF_EVEN); - } - - public List getCredits() { - return credits; - } - - public void setCredits(List credits) { - this.credits = credits; + public QuotaBalanceResponse() { + super("balance"); } - public void addCredits(QuotaBalanceVO credit) { - QuotaCreditsResponse cr = new QuotaCreditsResponse(); - cr.setCredit(credit.getCreditBalance()); - cr.setCreditedOn(credit.getUpdatedOn()); - credits.add(0, cr); + public QuotaBalanceResponse(Date date, BigDecimal balance) { + super("balance"); + this.date = date; + this.balance = balance; } - public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + public void setCurrency(String currency) { + this.currency = currency; } - public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); + public void setBalances(List balances) { + this.balances = balances; } - public Date getEndDate() { - return endDate == null ? null : new Date(endDate.getTime()); + public String getCurrency() { + return currency; } - public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); + public List getBalances() { + return balances; } - public String getCurrency() { - return currency; + public Date getDate() { + return date; } - public void setCurrency(String currency) { - this.currency = currency; + public BigDecimal getBalance() { + return balance; } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java index bde905c487b7..f0390bf626d0 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java @@ -29,7 +29,6 @@ import org.apache.cloudstack.api.command.QuotaTariffListCmd; import org.apache.cloudstack.api.command.QuotaTariffUpdateCmd; import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; -import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; @@ -50,14 +49,10 @@ public interface QuotaResponseBuilder { QuotaStatementResponse createQuotaStatementResponse(QuotaStatementCmd cmd); - QuotaBalanceResponse createQuotaBalanceResponse(List quotaUsage, Date startDate, Date endDate); + QuotaBalanceResponse createQuotaBalanceResponse(QuotaBalanceCmd cmd); Pair, Integer> createQuotaSummaryResponse(QuotaSummaryCmd cmd); - QuotaBalanceResponse createQuotaLastBalanceResponse(List quotaBalance, Date startDate); - - List getQuotaBalance(QuotaBalanceCmd cmd); - QuotaCreditsResponse addQuotaCredits(Long accountId, Long domainId, Double amount, Long updatedBy, Boolean enforce); List listQuotaEmailTemplates(QuotaEmailTemplateListCmd cmd); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java index c919bb5887c1..231ba777e600 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java @@ -26,13 +26,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; -import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; -import java.util.Iterator; import java.util.List; -import java.util.ListIterator; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -329,93 +326,18 @@ public boolean isUserAllowedToSeeActivationRules(User user) { } @Override - public QuotaBalanceResponse createQuotaBalanceResponse(List quotaBalance, Date startDate, Date endDate) { - if (quotaBalance == null || quotaBalance.isEmpty()) { - throw new InvalidParameterValueException("The request period does not contain balance entries."); - } - Collections.sort(quotaBalance, new Comparator() { - @Override - public int compare(QuotaBalanceVO o1, QuotaBalanceVO o2) { - o1 = o1 == null ? new QuotaBalanceVO() : o1; - o2 = o2 == null ? new QuotaBalanceVO() : o2; - return o2.getUpdatedOn().compareTo(o1.getUpdatedOn()); // desc - } - }); - - boolean have_balance_entries = false; - //check that there is at least one balance entry - for (Iterator it = quotaBalance.iterator(); it.hasNext();) { - QuotaBalanceVO entry = it.next(); - if (entry.isBalanceEntry()) { - have_balance_entries = true; - break; - } - } - //if last entry is a credit deposit then remove that as that is already - //accounted for in the starting balance after that entry, note the sort is desc - if (have_balance_entries) { - ListIterator li = quotaBalance.listIterator(quotaBalance.size()); - // Iterate in reverse. - while (li.hasPrevious()) { - QuotaBalanceVO entry = li.previous(); - if (logger.isDebugEnabled()) { - logger.debug("createQuotaBalanceResponse: Entry=" + entry); - } - if (entry.getCreditsId() > 0) { - li.remove(); - } else { - break; - } - } - } + public QuotaBalanceResponse createQuotaBalanceResponse(QuotaBalanceCmd cmd) { + List quotaBalances = _quotaService.listQuotaBalancesForAccount(cmd.getEntityOwnerId(), cmd.getStartDate(), cmd.getEndDate()); - int quota_activity = quotaBalance.size(); - QuotaBalanceResponse resp = new QuotaBalanceResponse(); - BigDecimal lastCredits = new BigDecimal(0); - boolean consecutive = true; - for (Iterator it = quotaBalance.iterator(); it.hasNext();) { - QuotaBalanceVO entry = it.next(); - if (logger.isDebugEnabled()) { - logger.debug("createQuotaBalanceResponse: All Credit Entry=" + entry); - } - if (entry.getCreditsId() > 0) { - if (consecutive) { - lastCredits = lastCredits.add(entry.getCreditBalance()); - } - resp.addCredits(entry); - it.remove(); - } else { - consecutive = false; - } - } + List balances = quotaBalances.stream() + .map(balance -> new QuotaBalanceResponse(balance.getUpdatedOn(), balance.getCreditBalance())) + .collect(Collectors.toList()); - if (quota_activity > 0 && quotaBalance.size() > 0) { - // order is desc last item is the start item - QuotaBalanceVO startItem = quotaBalance.get(quotaBalance.size() - 1); - QuotaBalanceVO endItem = quotaBalance.get(0); - resp.setStartDate(startDate); - resp.setStartQuota(startItem.getCreditBalance()); - resp.setEndDate(endDate); - if (logger.isDebugEnabled()) { - logger.debug("createQuotaBalanceResponse: Start Entry=" + startItem); - logger.debug("createQuotaBalanceResponse: End Entry=" + endItem); - } - resp.setEndQuota(endItem.getCreditBalance().add(lastCredits)); - } else if (quota_activity > 0) { - // order is desc last item is the start item - resp.setStartDate(startDate); - resp.setStartQuota(new BigDecimal(0)); - resp.setEndDate(endDate); - resp.setEndQuota(new BigDecimal(0).add(lastCredits)); - } else { - resp.setStartDate(startDate); - resp.setEndDate(endDate); - resp.setStartQuota(new BigDecimal(0)); - resp.setEndQuota(new BigDecimal(0)); - } - resp.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); - resp.setObjectName("balance"); - return resp; + QuotaBalanceResponse response = new QuotaBalanceResponse(); + response.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); + response.setBalances(balances); + + return response; } @Override @@ -784,7 +706,7 @@ public QuotaCreditsResponse addQuotaCredits(Long accountId, Long domainId, Doubl throw new InvalidParameterValueException("Account does not exist with account id " + accountId); } final boolean lockAccountEnforcement = "true".equalsIgnoreCase(QuotaConfig.QuotaEnableEnforcement.value()); - final BigDecimal currentAccountBalance = _quotaBalanceDao.lastQuotaBalance(accountId, domainId, startOfNextDay(new Date(depositedOn.getTime()))); + final BigDecimal currentAccountBalance = _quotaBalanceDao.getLastQuotaBalance(accountId, domainId); logger.debug("Depositing [{}] credits on adjusted date [{}]; current balance is [{}].", amount, DateUtil.displayDateInTimezone(QuotaManagerImpl.getUsageAggregationTimeZone(), depositedOn), currentAccountBalance); // update quota account with the balance @@ -848,34 +770,6 @@ public boolean updateQuotaEmailTemplate(QuotaEmailTemplateUpdateCmd cmd) { return false; } - @Override - public QuotaBalanceResponse createQuotaLastBalanceResponse(List quotaBalance, Date startDate) { - if (quotaBalance == null) { - throw new InvalidParameterValueException("There are no balance entries on or before the requested date."); - } - if (startDate == null) { - startDate = new Date(); - } - QuotaBalanceResponse resp = new QuotaBalanceResponse(); - BigDecimal lastCredits = new BigDecimal(0); - for (QuotaBalanceVO entry : quotaBalance) { - logger.debug("createQuotaLastBalanceResponse Date={} balance={} credit={}", - DateUtil.displayDateInTimezone(QuotaManagerImpl.getUsageAggregationTimeZone(), entry.getUpdatedOn()), - entry.getCreditBalance(), entry.getCreditsId()); - - lastCredits = lastCredits.add(entry.getCreditBalance()); - } - resp.setStartQuota(lastCredits); - resp.setStartDate(startDate); - resp.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); - resp.setObjectName("balance"); - return resp; - } - - @Override - public List getQuotaBalance(QuotaBalanceCmd cmd) { - return _quotaService.findQuotaBalanceVO(cmd.getAccountId(), cmd.getAccountName(), cmd.getDomainId(), cmd.getStartDate(), cmd.getEndDate()); - } @Override public Date startOfNextDay(Date date) { LocalDate localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java index 78acfc11682e..478e43d2e203 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java @@ -30,7 +30,7 @@ public interface QuotaService extends PluggableService { List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate); - List findQuotaBalanceVO(Long accountId, String accountName, Long domainId, Date startDate, Date endDate); + List listQuotaBalancesForAccount(Long accountId, Date startDate, Date endDate); void setLockAccount(Long accountId, Boolean state); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java index a0ba2fbc751d..e1ec2ebb2a60 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java @@ -28,6 +28,7 @@ import com.cloud.projects.ProjectManager; import com.cloud.user.AccountService; +import com.cloud.utils.DateUtil; import org.apache.cloudstack.api.command.QuotaBalanceCmd; import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaCreditsCmd; @@ -58,17 +59,16 @@ import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.time.DateUtils; import org.springframework.stereotype.Component; import com.cloud.configuration.Config; import com.cloud.domain.dao.DomainDao; import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.PermissionDeniedException; import com.cloud.user.Account; import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; import com.cloud.utils.component.ManagerBase; -import com.cloud.utils.db.Filter; @Component public class QuotaServiceImpl extends ManagerBase implements QuotaService, Configurable, QuotaConfig { @@ -152,63 +152,53 @@ public Boolean isQuotaServiceEnabled() { } @Override - public List findQuotaBalanceVO(Long accountId, String accountName, Long domainId, Date startDate, Date endDate) { - if ((accountId == null) && (accountName != null) && (domainId != null)) { - Account userAccount = null; - Account caller = CallContext.current().getCallingAccount(); - if (_domainDao.isChildDomain(caller.getDomainId(), domainId)) { - Filter filter = new Filter(AccountVO.class, "id", Boolean.FALSE, null, null); - List accounts = _accountDao.listAccounts(accountName, domainId, filter); - if (!accounts.isEmpty()) { - userAccount = accounts.get(0); - } - if (userAccount != null) { - accountId = userAccount.getId(); - } else { - throw new InvalidParameterValueException("Unable to find account " + accountName + " in domain " + domainId); - } - } else { - throw new PermissionDeniedException("Invalid Domain Id or Account"); - } + public List listQuotaBalancesForAccount(Long accountId, Date startDate, Date endDate) { + validateStartDateAndEndDateForListQuotaBalancesForAccount(startDate, endDate); + + if (accountId == -1) { + accountId = CallContext.current().getCallingAccountId(); } + Account account = _accountDao.findByIdIncludingRemoved(accountId); + Long domainId = account.getDomainId(); - startDate = startDate == null ? new Date() : startDate; + if (startDate == null && endDate == null) { + logger.debug("Retrieving last quota balance for {}.", account); + QuotaBalanceVO lastQuotaBalance = _quotaBalanceDao.getLastQuotaBalanceEntry(accountId, domainId, null); - if (endDate == null) { - // adjust start date to end of day as there is no end date - startDate = _respBldr.startOfNextDay(startDate); - if (logger.isDebugEnabled()) { - logger.debug("getQuotaBalance1: Getting quota balance records for account: " + accountId + ", domainId: " + domainId + ", on or before " + startDate); - } - List qbrecords = _quotaBalanceDao.lastQuotaBalanceVO(accountId, domainId, startDate); - if (logger.isDebugEnabled()) { - logger.debug("Found records size=" + qbrecords.size()); - } - if (qbrecords.isEmpty()) { - logger.info("Incorrect Date there are no quota records before this date " + startDate); - return qbrecords; - } else { - return qbrecords; - } - } else { - if (startDate.before(endDate)) { - if (logger.isDebugEnabled()) { - logger.debug("getQuotaBalance2: Getting quota balance records for account: " + accountId + ", domainId: " + domainId + ", between " + startDate - + " and " + endDate); - } - List qbrecords = _quotaBalanceDao.findQuotaBalance(accountId, domainId, startDate, endDate); - if (logger.isDebugEnabled()) { - logger.debug("getQuotaBalance3: Found records size=" + qbrecords.size()); - } - if (qbrecords.isEmpty()) { - logger.info("There are no quota records between these dates start date " + startDate + " and end date:" + endDate); - return qbrecords; - } else { - return qbrecords; - } - } else { - throw new InvalidParameterValueException("Incorrect Date Range. Start date: " + startDate + " is after end date:" + endDate); + if (lastQuotaBalance == null) { + logger.debug("Did not found a quota balance entry for {}.", account); + return new ArrayList<>(); } + + return List.of(lastQuotaBalance); + } + + if (endDate == null) { + endDate = DateUtils.addDays(new Date(), -1); + } + + List quotaBalances = _quotaBalanceDao.listQuotaBalances(accountId, domainId, startDate, endDate); + + if (quotaBalances.isEmpty()) { + logger.info("There are no quota balances for {} between [{}] and [{}].", account, + DateUtil.getOutputString(startDate), DateUtil.getOutputString(endDate)); + } + + return quotaBalances; + } + + protected void validateStartDateAndEndDateForListQuotaBalancesForAccount(Date startDate, Date endDate) { + if (startDate == null && endDate != null) { + throw new InvalidParameterValueException("Parameter \"enddate\" must be informed together with parameter \"startdate\"."); + } + + Date now = new Date(); + if (startDate != null && startDate.after(now)) { + throw new InvalidParameterValueException("The last balance can be at most from yesterday; therefore, the start date must be before today."); + } + + if (ObjectUtils.allNotNull(startDate, endDate) && startDate.after(endDate)) { + throw new InvalidParameterValueException("The start date cannot be after the end date."); } } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaBalanceCmdTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaBalanceCmdTest.java index adabc694f25a..cd0eeb2c3b48 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaBalanceCmdTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaBalanceCmdTest.java @@ -16,18 +16,15 @@ // under the License. package org.apache.cloudstack.api.command; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - import org.apache.cloudstack.api.response.QuotaBalanceResponse; import org.apache.cloudstack.api.response.QuotaResponseBuilder; -import org.apache.cloudstack.quota.vo.QuotaBalanceVO; +import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; import junit.framework.TestCase; @@ -36,31 +33,20 @@ public class QuotaBalanceCmdTest extends TestCase { @Mock - QuotaResponseBuilder responseBuilder; + QuotaResponseBuilder quotaResponseBuilderMock; - @Test - public void testQuotaBalanceCmd() throws NoSuchFieldException, IllegalAccessException { - QuotaBalanceCmd cmd = new QuotaBalanceCmd(); - Field rbField = QuotaBalanceCmd.class.getDeclaredField("_responseBuilder"); - rbField.setAccessible(true); - rbField.set(cmd, responseBuilder); + @InjectMocks + @Spy + QuotaBalanceCmd quotaBalanceCmdSpy; - List quotaBalanceVOList = new ArrayList(); - Mockito.when(responseBuilder.getQuotaBalance(Mockito.any(cmd.getClass()))).thenReturn(quotaBalanceVOList); - Mockito.when(responseBuilder.createQuotaLastBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class))).thenReturn(new QuotaBalanceResponse()); - Mockito.when(responseBuilder.createQuotaBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class), Mockito.any(Date.class))).thenReturn(new QuotaBalanceResponse()); - Mockito.lenient().when(responseBuilder.startOfNextDay(Mockito.any(Date.class))).thenReturn(new Date()); + @Test + public void executeTestSetResponseObject() { + QuotaBalanceResponse expected = new QuotaBalanceResponse(); + Mockito.doReturn(expected).when(quotaResponseBuilderMock).createQuotaBalanceResponse(Mockito.eq(quotaBalanceCmdSpy)); - // end date not specified - cmd.setStartDate(new Date()); - cmd.setEndDate(null); - cmd.execute(); - Mockito.verify(responseBuilder, Mockito.times(1)).createQuotaLastBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class)); - Mockito.verify(responseBuilder, Mockito.times(0)).createQuotaBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class), Mockito.any(Date.class)); + quotaBalanceCmdSpy.execute(); - // end date specified - cmd.setEndDate(new Date()); - cmd.execute(); - Mockito.verify(responseBuilder, Mockito.times(1)).createQuotaBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class), Mockito.any(Date.class)); + Assert.assertEquals(expected, quotaBalanceCmdSpy.getResponseObject()); } + } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java index 81b4992082d9..0c7610461c15 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java @@ -40,6 +40,7 @@ import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.QuotaBalanceCmd; import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaCreditsListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; @@ -249,7 +250,7 @@ public void testAddQuotaCredits() { credit.setCredit(new BigDecimal(amount)); Mockito.when(quotaCreditsDaoMock.saveCredits(Mockito.any(QuotaCreditsVO.class))).thenReturn(credit); - Mockito.when(quotaBalanceDaoMock.lastQuotaBalance(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(Date.class))).thenReturn(new BigDecimal(111)); + Mockito.when(quotaBalanceDaoMock.getLastQuotaBalance(Mockito.anyLong(), Mockito.anyLong())).thenReturn(new BigDecimal(111)); Mockito.doReturn(userVoMock).when(quotaResponseBuilderSpy).getCreditorForQuotaCredits(credit); AccountVO account = new AccountVO(); @@ -295,33 +296,6 @@ public void testUpdateQuotaEmailTemplate() { assertTrue(quotaResponseBuilderSpy.updateQuotaEmailTemplate(cmd)); } - @Test - public void testCreateQuotaLastBalanceResponse() { - List quotaBalance = new ArrayList<>(); - // null balance test - try { - quotaResponseBuilderSpy.createQuotaLastBalanceResponse(null, new Date()); - } catch (InvalidParameterValueException e) { - assertTrue(e.getMessage().equals("There are no balance entries on or before the requested date.")); - } - - // empty balance test - try { - quotaResponseBuilderSpy.createQuotaLastBalanceResponse(quotaBalance, new Date()); - } catch (InvalidParameterValueException e) { - assertTrue(e.getMessage().equals("There are no balance entries on or before the requested date.")); - } - - // valid balance test - QuotaBalanceVO entry = new QuotaBalanceVO(); - entry.setAccountId(2L); - entry.setCreditBalance(new BigDecimal(100)); - quotaBalance.add(entry); - quotaBalance.add(entry); - QuotaBalanceResponse resp = quotaResponseBuilderSpy.createQuotaLastBalanceResponse(quotaBalance, null); - assertTrue(resp.getStartQuota().compareTo(new BigDecimal(200)) == 0); - } - @Test public void testStartOfNextDayWithoutParameters() { Date nextDate = quotaResponseBuilderSpy.startOfNextDay(); @@ -919,6 +893,46 @@ public void injectUsageTypeVariablesTestReturnInjectedVariables() { Assert.assertTrue(formattedVariables.containsValue("zonename")); } + private List getQuotaBalancesForTest() { + List balances = new ArrayList<>(); + + QuotaBalanceVO balance = new QuotaBalanceVO(); + balance.setUpdatedOn(new Date()); + balance.setCreditBalance(BigDecimal.valueOf(-10.42)); + balances.add(balance); + + balance = new QuotaBalanceVO(); + balance.setUpdatedOn(new Date()); + balance.setCreditBalance(BigDecimal.valueOf(-18.94)); + balances.add(balance); + + balance = new QuotaBalanceVO(); + balance.setUpdatedOn(new Date()); + balance.setCreditBalance(BigDecimal.valueOf(-29.37)); + balances.add(balance); + + return balances; + } + + @Test + public void createQuotaBalancesResponseTestCreateResponse() { + List balances = getQuotaBalancesForTest(); + + QuotaBalanceResponse expected = new QuotaBalanceResponse(); + expected.setObjectName("balance"); + expected.setCurrency("$"); + + Mockito.doReturn(balances).when(quotaServiceMock).listQuotaBalancesForAccount(Mockito.any(), Mockito.any(), Mockito.any()); + QuotaBalanceResponse result = quotaResponseBuilderSpy.createQuotaBalanceResponse(new QuotaBalanceCmd()); + + Assert.assertEquals(expected.getCurrency(), result.getCurrency()); + + for (int i = 0; i < balances.size(); i++) { + Assert.assertEquals(balances.get(i).getUpdatedOn(), result.getBalances().get(i).getDate()); + Assert.assertEquals(balances.get(i).getCreditBalance(), result.getBalances().get(i).getBalance()); + } + } + @Test public void createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformedTestUsageTypeDifferentFromNullDoNothing() { List listUsage = new ArrayList<>(); diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java index a0fe63de8518..89c48b1cc660 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java @@ -18,6 +18,7 @@ import com.cloud.configuration.Config; import com.cloud.domain.dao.DomainDao; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; import com.cloud.utils.db.TransactionLegacy; @@ -31,7 +32,9 @@ import org.apache.cloudstack.quota.dao.QuotaUsageJoinDao; import org.apache.cloudstack.quota.vo.QuotaAccountVO; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; +import org.apache.commons.lang3.time.DateUtils; import org.joda.time.DateTime; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -43,7 +46,6 @@ import javax.naming.ConfigurationException; import java.lang.reflect.Field; -import java.math.BigDecimal; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -51,6 +53,8 @@ @RunWith(MockitoJUnitRunner.class) public class QuotaServiceImplTest extends TestCase { + @Mock + AccountVO accountVoMock; @Mock AccountDao accountDaoMock; @Mock @@ -67,9 +71,6 @@ public class QuotaServiceImplTest extends TestCase { QuotaUsageJoinDao quotaUsageJoinDaoMock; @Mock QuotaResponseBuilder respBldr; - @Mock - private AccountVO accountVoMock; - @Spy @InjectMocks QuotaServiceImpl quotaServiceImplSpy; @@ -112,30 +113,6 @@ public void setup() throws IllegalAccessException, NoSuchFieldException, Configu quotaServiceImplSpy.configure("randomName", null); } - @Test - public void testFindQuotaBalanceVO() { - final long accountId = 2L; - final String accountName = "admin123"; - final long domainId = 1L; - final Date startDate = new DateTime().minusDays(2).toDate(); - final Date endDate = new Date(); - - List records = new ArrayList<>(); - QuotaBalanceVO qb = new QuotaBalanceVO(); - qb.setCreditBalance(new BigDecimal(100)); - qb.setAccountId(accountId); - records.add(qb); - - Mockito.when(respBldr.startOfNextDay(Mockito.any(Date.class))).thenReturn(startDate); - Mockito.when(quotaBalanceDao.findQuotaBalance(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.any(Date.class), Mockito.any(Date.class))).thenReturn(records); - Mockito.when(quotaBalanceDao.lastQuotaBalanceVO(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.any(Date.class))).thenReturn(records); - - // with enddate - assertTrue(quotaServiceImplSpy.findQuotaBalanceVO(accountId, accountName, domainId, startDate, endDate).get(0).equals(qb)); - // without enddate - assertTrue(quotaServiceImplSpy.findQuotaBalanceVO(accountId, accountName, domainId, startDate, null).get(0).equals(qb)); - } - @Test public void testGetQuotaUsage() { final long accountId = 2L; @@ -182,4 +159,66 @@ public void testSetMinBalance() { Mockito.verify(quotaAcc, Mockito.times(1)).persistQuotaAccount(Mockito.any(QuotaAccountVO.class)); } + @Test(expected = InvalidParameterValueException.class) + public void validateStartDateAndEndDateForListQuotaBalancesForAccountTestStartDateIsNullAndEndDateIsNotNullThrowsInvalidParameterException() { + quotaServiceImplSpy.validateStartDateAndEndDateForListQuotaBalancesForAccount(null, new Date()); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateStartDateAndEndDateForListQuotaBalancesForAccountTestStartDateIsAfterNowThrowsInvalidParameterValueException() { + Date startDate = DateUtils.addMinutes(new Date(), 1); + quotaServiceImplSpy.validateStartDateAndEndDateForListQuotaBalancesForAccount(startDate, null); + } + + @Test + public void validateStartDateAndEndDateForListQuotaBalancesForAccountTestEndDateIsAfterNowDoesNothing() { + Date startDate = DateUtils.addMinutes(new Date(), -1); + Date endDate = DateUtils.addMinutes(new Date(), 1); + quotaServiceImplSpy.validateStartDateAndEndDateForListQuotaBalancesForAccount(startDate, endDate); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateStartDateAndEndDateForListQuotaBalancesForAccountTestStartDateIsAfterEndDateThrowsInvalidParameterValueException() { + Date startDate = DateUtils.addMinutes(new Date(), -10); + Date endDate = DateUtils.addMinutes(new Date(), -15); + quotaServiceImplSpy.validateStartDateAndEndDateForListQuotaBalancesForAccount(startDate, endDate); + } + + @Test + public void listQuotaBalancesForAccountTestLastQuotaBalanceIsNullReturnsEmptyList() { + Mockito.doNothing().when(quotaServiceImplSpy).validateStartDateAndEndDateForListQuotaBalancesForAccount(Mockito.any(), Mockito.any()); + Mockito.doReturn(null).when(quotaBalanceDao).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(Mockito.mock(AccountVO.class)).when(accountDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + List result = quotaServiceImplSpy.listQuotaBalancesForAccount(1L, null, null); + + Assert.assertTrue(result.isEmpty()); + } + + @Test + public void listQuotaBalancesForAccountTestLastQuotaBalanceIsNotNullReturnsIt() { + QuotaBalanceVO expected = new QuotaBalanceVO(); + + Mockito.doNothing().when(quotaServiceImplSpy).validateStartDateAndEndDateForListQuotaBalancesForAccount(Mockito.any(), Mockito.any()); + Mockito.doReturn(expected).when(quotaBalanceDao).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(Mockito.mock(AccountVO.class)).when(accountDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + List result = quotaServiceImplSpy.listQuotaBalancesForAccount(1L, null, null); + + Assert.assertEquals(expected, result.get(0)); + } + + @Test + public void listQuotaBalancesForAccountTestReturnsQuotaBalances() { + List expected = new ArrayList<>(); + + Mockito.doNothing().when(quotaServiceImplSpy).validateStartDateAndEndDateForListQuotaBalancesForAccount(Mockito.any(), Mockito.any()); + Mockito.doReturn(expected).when(quotaBalanceDao).listQuotaBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + Mockito.doReturn(Mockito.mock(AccountVO.class)).when(accountDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + List result = quotaServiceImplSpy.listQuotaBalancesForAccount(1L, new Date(), null); + + Assert.assertEquals(expected, result); + } + } diff --git a/server/src/main/java/com/cloud/user/AccountManagerImpl.java b/server/src/main/java/com/cloud/user/AccountManagerImpl.java index d4d48bdf77be..52ec26661c9d 100644 --- a/server/src/main/java/com/cloud/user/AccountManagerImpl.java +++ b/server/src/main/java/com/cloud/user/AccountManagerImpl.java @@ -3922,7 +3922,7 @@ public Long finalizeAccountId(Long accountId, String accountName, Long domainId, if (getActiveAccountById(accountId) != null) { return accountId; } - throw new InvalidParameterValueException(String.format("Unable to find account with ID [%s].", accountId)); + throw new InvalidParameterValueException(String.format("Unable to find account with the specified ID.")); } if (accountName == null && domainId == null) { @@ -3938,16 +3938,16 @@ public Long finalizeAccountId(Long accountId, String accountName, Long domainId, throw new ServerApiException(ApiErrorCode.PARAM_ERROR, String.format("Both %s and %s are needed if using either. Consider using %s instead.", ApiConstants.ACCOUNT, ApiConstants.DOMAIN_ID, ApiConstants.ACCOUNT_ID)); } - throw new InvalidParameterValueException(String.format("Unable to find account by name [%s] on domain [%s].", accountName, domainId)); + throw new InvalidParameterValueException(String.format("Unable to find account with name [%s] on the specified domain.", accountName)); } protected long getActiveProjectAccountByProjectId(long projectId) { Project project = _projectMgr.getProject(projectId); if (project == null) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, String.format("Unable to find project with ID [%s].", projectId)); + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find project with the specified ID."); } if (project.getState() != Project.State.Active) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, String.format("Project with ID [%s] is not active.", projectId)); + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Project is not active."); } return project.getProjectAccountId(); } diff --git a/test/integration/plugins/quota/test_quota_balance.py b/test/integration/plugins/quota/test_quota_balance.py index f5c1c75d7b23..ca702bbda013 100644 --- a/test/integration/plugins/quota/test_quota_balance.py +++ b/test/integration/plugins/quota/test_quota_balance.py @@ -117,9 +117,28 @@ def tearDown(self): def delete_tariffs(self): for tariff in self.tariffs: cmd = quotaTariffDelete.quotaTariffDeleteCmd() - cmd.id = tariff.uuid + cmd.id = tariff.id self.apiclient.quotaTariffDelete(cmd) + def insert_usage_and_update_quota(self, zone_id, account_id, domain_id, relative_start_date, relative_end_date): + start_date = f"DATE_ADD(UTC_TIMESTAMP(), INTERVAL {relative_start_date} HOUR)" + end_date = f"DATE_ADD(UTC_TIMESTAMP(), INTERVAL {relative_end_date} HOUR)" + + # Manually insert a usage regarding the usage type 21 (VM_DISK_IO_READ) + sql_query = (f"INSERT INTO cloud_usage.cloud_usage (zone_id,account_id,domain_id,description,usage_display,usage_type,raw_usage,vm_instance_id,vm_name,offering_id,template_id," + f"usage_id,`type`,`size`,network_id,start_date,end_date,virtual_size,cpu_speed,cpu_cores,memory,quota_calculated,is_hidden,state)" + f" VALUES ('{zone_id}','{account_id}','{domain_id}','Test','1 Hrs',21,1,NULL,NULL,NULL,NULL,NULL,'VirtualMachine',NULL,NULL,{start_date},{end_date},NULL,NULL,NULL,NULL,0,0,NULL);") + self.debug(sql_query) + self.dbclient.execute(sql_query) + + # Update quota to calculate the balance of the account + cmd = quotaUpdate.quotaUpdateCmd() + self.apiclient.quotaUpdate(cmd) + time.sleep(1) + + def format_date(self, date): + return date.strftime("%Y-%m-%d %H:%M:%S") + @attr(tags=["advanced", "smoke", "quota"], required_hardware="false") def test_quota_balance(self): """ @@ -146,6 +165,7 @@ def test_quota_balance(self): cmd.domainid = self.domain.id cmd.value = 100 self.apiclient.quotaCredits(cmd) + time.sleep(1) # Fetch account ID from account_uuid account_id_select = f"SELECT id FROM account WHERE uuid = '{self.account.id}';" @@ -165,27 +185,24 @@ def test_quota_balance(self): qresultset = self.dbclient.execute(zone_id_select) zone_id = qresultset[0][0] - start_date = datetime.datetime.now() + datetime.timedelta(seconds=1) - end_date = datetime.datetime.now() + datetime.timedelta(hours=1) - - # Manually insert a usage regarding the usage type 21 (VM_DISK_IO_READ) - sql_query = (f"INSERT INTO cloud_usage.cloud_usage (zone_id,account_id,domain_id,description,usage_display,usage_type,raw_usage,vm_instance_id,vm_name,offering_id,template_id," - f"usage_id,`type`,`size`,network_id,start_date,end_date,virtual_size,cpu_speed,cpu_cores,memory,quota_calculated,is_hidden,state)" - f" VALUES ('{zone_id}','{account_id}','{domain_id}','Test','1 Hrs',21,1,NULL,NULL,NULL,NULL,NULL,'VirtualMachine',NULL,NULL,'{start_date}','{end_date}',NULL,NULL,NULL,NULL,0,0,NULL);") - self.debug(sql_query) - self.dbclient.execute(sql_query) - - # Update quota to calculate the balance of the account - cmd = quotaUpdate.quotaUpdateCmd() - self.apiclient.quotaUpdate(cmd) + # Generate three quota_balance entries + self.insert_usage_and_update_quota(zone_id, account_id, domain_id, 0, 1) + self.insert_usage_and_update_quota(zone_id, account_id, domain_id, 1, 2) + self.insert_usage_and_update_quota(zone_id, account_id, domain_id, 2, 3) # Retrieve the quota balance of the account cmd = quotaBalance.quotaBalanceCmd() cmd.domainid = self.account.domainid cmd.account = self.account.name + cmd.startdate = datetime.datetime.now() + datetime.timedelta(hours=-1) + cmd.enddate = datetime.datetime.now() + datetime.timedelta(hours=3) response = self.apiclient.quotaBalance(cmd) - self.debug(f"The quota balance for the account {self.account.name} is {response.balance}.") - self.assertEqual(response.balance.startquota, 90, f"The `startQuota` response field is supposed to be 90 but was {response.balance.startquota}.") + self.assertTrue(len(response.balance.balances) == 4, f"Expected 4 balance entries for between {self.format_date(cmd.startdate)} " + + f"and {self.format_date(cmd.enddate)} but got {len(response.balance.balances)}.") + for i, balance in enumerate(response.balance.balances): + expected_balance = 100 - 10 * i + self.debug(f"The quota balance for the account {self.account.name} at {balance.date} was {balance.balance}.") + self.assertEqual(balance.balance, expected_balance, f"The `balance` field at {balance.date} is supposed to be {expected_balance} but was {balance.balance}.") return From 6786d45951a5faf26ce3caa7fffecb880bbfd5e0 Mon Sep 17 00:00:00 2001 From: James Peru Mmbono Date: Wed, 27 May 2026 15:09:32 +0300 Subject: [PATCH 07/28] fix(linstor): surface ambiguous template fallbacks and legacy orphan cleanup (#13078) Co-authored-by: jmsperu --- .../kvm/storage/LinstorStorageAdaptor.java | 14 +++++++++++-- .../storage/datastore/util/LinstorUtil.java | 21 ++++++++++++++++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java index 31a41cd94079..32858108d6ac 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java @@ -508,9 +508,19 @@ private boolean deRefOrDeleteResource(DevelopersApi api, String rscName, String // if there is only one template-for property left for templates, the template isn't needed anymore // or if it isn't a template anyway, it will not have this Aux property // _cs-template-for- properties work like a ref-count. - if (rd.getProps().keySet().stream() + long remainingTemplateRefs = rd.getProps().keySet().stream() .filter(key -> key.startsWith("Aux/" + LinstorUtil.CS_TEMPLATE_FOR_PREFIX)) - .count() == expectedProps) { + .count(); + if (remainingTemplateRefs == expectedProps) { + // Surface the legacy case where a resource has zero `_cs-template-for-` aux + // properties even though we never decremented one — that's a template predating + // the ref-count convention, or a stale orphan. Logging before deletion lets + // operators audit how many such orphans existed at upgrade time. + if (expectedProps == 0) { + logger.info("Linstor: deleting resource {} which has no _cs-template-for- aux properties " + + "(legacy template predating the ref-count convention, or a stale orphan). " + + "Resource group context: {}", rd.getName(), rscGrpName); + } ApiCallRcList answers = api.resourceDefinitionDelete(rd.getName()); checkLinstorAnswersThrow(answers); deleted = true; diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index 239331077e1b..502bf6d0ca2a 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -543,7 +543,26 @@ public static ResourceDefinition findResourceDefinition(DevelopersApi api, Strin .filter(rscDfn -> rscDfn.getProps().containsKey(LinstorUtil.getTemplateForAuxPropKey(rscGrpName))) .findFirst(); - return rd.orElseGet(() -> rdsStartingWith.get(0)); + if (rd.isPresent()) { + return rd.get(); + } + // Fallback: no resource has the exact "_cs-template-for-" property. + // This happens when (a) the matched resource is a legacy template created before that + // convention was introduced, or (b) the template was cached by a different resource + // group and the operator hopes to share it. Log so the ambiguity is visible — silent + // first-match fallback has previously routed clones to the wrong template when + // multiple resource groups coexisted on the same controller. + ResourceDefinition fallback = rdsStartingWith.get(0); + LOGGER.warn("LINSTOR findResourceDefinition: no resource for '{}' has the expected " + + "Aux property '{}' for resource group '{}'; falling back to first match '{}' " + + "(present aux properties: {}). If this is wrong, set the property explicitly " + + "or remove the unrelated resource definition.", + rscName, getTemplateForAuxPropKey(rscGrpName), rscGrpName, + fallback.getName(), + fallback.getProps().keySet().stream() + .filter(k -> k.startsWith("Aux/" + CS_TEMPLATE_FOR_PREFIX)) + .collect(Collectors.toList())); + return fallback; } public static boolean isRscDiskless(ResourceWithVolumes rsc) { From e3245515e8bb758f3c1278f15077231271a141e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sil=C3=A9n?= Date: Wed, 27 May 2026 23:13:11 +0300 Subject: [PATCH 08/28] docs: note MariaDB support in README (#13238) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a5aacb49f6b5..852674ab7b1f 100644 --- a/README.md +++ b/README.md @@ -203,7 +203,7 @@ Unrestricted (TSU) exception (see the BIS Export Administration Regulations, Sec The following provides more details on the included cryptographic software: * CloudStack makes use of JaSypt cryptographic libraries. -* CloudStack has a system requirement of MySQL, and uses native database encryption functionality. +* CloudStack requires a MySQL-compatible database (MariaDB or MySQL), and uses native database encryption functionality. * CloudStack makes use of the Bouncy Castle general-purpose encryption library. * CloudStack can optionally interact with and control OpenSwan-based VPNs. * CloudStack has a dependency on and makes use of JSch - a java SSH2 implementation. From c0ce5b461f816bd6a6c1ceee8c1ee3104b3dbde7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 21:35:13 -0300 Subject: [PATCH 09/28] Bump the github-actions-dependencies group across 1 directory with 2 updates (#13215) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/daily-repo-status.lock.yml | 10 +++++----- .github/workflows/issue-triage-agent.lock.yml | 10 +++++----- .github/workflows/stale.yml | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/daily-repo-status.lock.yml b/.github/workflows/daily-repo-status.lock.yml index f8ce13c3cee8..c9b85fafafdd 100644 --- a/.github/workflows/daily-repo-status.lock.yml +++ b/.github/workflows/daily-repo-status.lock.yml @@ -54,7 +54,7 @@ jobs: comment_repo: "" steps: - name: Setup - uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.45. + uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.45. with: destination: /opt/gh-aw/ - name: Check workflow file @@ -96,7 +96,7 @@ jobs: secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} steps: - name: Setup - uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.45. + uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.45. with: destination: /opt/gh-aw/ - name: Checkout @@ -784,7 +784,7 @@ jobs: total_count: ${{ steps.missing_tool.outputs.total_count }} steps: - name: Setup - uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.45. + uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.45. with: destination: /opt/gh-aw/ - name: Download agent output @@ -881,7 +881,7 @@ jobs: success: ${{ steps.parse_results.outputs.success }} steps: - name: Setup - uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.45. + uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.45. with: destination: /opt/gh-aw/ - name: Download agent @@ -993,7 +993,7 @@ jobs: process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} steps: - name: Setup - uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.45. + uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.45. with: destination: /opt/gh-aw/ - name: Download agent output diff --git a/.github/workflows/issue-triage-agent.lock.yml b/.github/workflows/issue-triage-agent.lock.yml index 1c3d11a800e6..258032f02c4a 100644 --- a/.github/workflows/issue-triage-agent.lock.yml +++ b/.github/workflows/issue-triage-agent.lock.yml @@ -53,7 +53,7 @@ jobs: comment_repo: "" steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.71.1 + uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.74.8 with: destination: /opt/gh-aw/actions - name: Check workflow file timestamps @@ -91,7 +91,7 @@ jobs: secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.71.1 + uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.74.8 with: destination: /opt/gh-aw/actions - name: Checkout repository @@ -780,7 +780,7 @@ jobs: total_count: ${{ steps.missing_tool.outputs.total_count }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.71.1 + uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.74.8 with: destination: /opt/gh-aw/actions - name: Download agent output artifact @@ -874,7 +874,7 @@ jobs: success: ${{ steps.parse_results.outputs.success }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.71.1 + uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.74.8 with: destination: /opt/gh-aw/actions - name: Download agent artifacts @@ -987,7 +987,7 @@ jobs: process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.71.1 + uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.74.8 with: destination: /opt/gh-aw/actions - name: Download agent output artifact diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 5c92f23888e2..c0da5f98cc8e 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -28,7 +28,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 + - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0 with: stale-issue-message: 'This issue is stale because it has been open for 120 days with no activity. It may be removed by administrators of this project at any time. Remove the stale label or comment to request for removal of it to prevent this.' stale-pr-message: 'This PR is stale because it has been open for 120 days with no activity. It may be removed by administrators of this project at any time. Remove the stale label or comment to request for removal of it to prevent this.' @@ -41,7 +41,7 @@ jobs: days-before-pr-close: 240 exempt-issue-labels: 'gsoc,good-first-issue,long-term-plan' exempt-pr-labels: 'status:ready-for-merge,status:needs-testing,status:on-hold' - - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 + - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0 with: stale-issue-label: 'archive' days-before-stale: 240 From 7308dad19a9a70d33a33a75bea8c51629541e438 Mon Sep 17 00:00:00 2001 From: Vishesh <8760112+vishesh92@users.noreply.github.com> Date: Fri, 29 May 2026 14:45:49 +0530 Subject: [PATCH 10/28] Update github aw actions (#13278) --- .github/aw/actions-lock.json | 56 + .../.github_workflows_shared_reporting.md | 73 + .github/dependabot.yml | 2 + .github/workflows/daily-repo-status.lock.yml | 1782 ++++++++++------- .github/workflows/daily-repo-status.md | 8 +- .github/workflows/issue-triage-agent.lock.yml | 1499 ++++++++------ .github/workflows/issue-triage-agent.md | 22 +- 7 files changed, 2167 insertions(+), 1275 deletions(-) create mode 100644 .github/aw/actions-lock.json create mode 100644 .github/aw/imports/github/gh-aw/359795d49ada21681ab616bd4cbcb144a7387115/.github_workflows_shared_reporting.md diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json new file mode 100644 index 000000000000..ea25ffab6b9e --- /dev/null +++ b/.github/aw/actions-lock.json @@ -0,0 +1,56 @@ +{ + "entries": { + "actions/github-script@v9.0.0": { + "repo": "actions/github-script", + "version": "v9.0.0", + "sha": "3a2844b7e9c422d3c10d287c895573f7108da1b3" + }, + "github/gh-aw-actions/setup@v0.76.1": { + "repo": "github/gh-aw-actions/setup", + "version": "v0.76.1", + "sha": "46d564922b082d0db93244972e8005ea6904ee5f" + } + }, + "containers": { + "ghcr.io/github/gh-aw-firewall/agent:0.18.0": { + "image": "ghcr.io/github/gh-aw-firewall/agent:0.18.0", + "digest": "sha256:ab84dfc7f5998cb8cd0c596526dd573b7e7d06c6a740266a1e6df879fa16c866", + "pinned_image": "ghcr.io/github/gh-aw-firewall/agent:0.18.0@sha256:ab84dfc7f5998cb8cd0c596526dd573b7e7d06c6a740266a1e6df879fa16c866" + }, + "ghcr.io/github/gh-aw-firewall/agent:0.25.55": { + "image": "ghcr.io/github/gh-aw-firewall/agent:0.25.55", + "digest": "sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731", + "pinned_image": "ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731" + }, + "ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55": { + "image": "ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55", + "digest": "sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3", + "pinned_image": "ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3" + }, + "ghcr.io/github/gh-aw-firewall/squid:0.18.0": { + "image": "ghcr.io/github/gh-aw-firewall/squid:0.18.0", + "digest": "sha256:82a5d062a5612a57a43a171a5b79ddbb690a86a8ddda02339cc1675131ae9f8b", + "pinned_image": "ghcr.io/github/gh-aw-firewall/squid:0.18.0@sha256:82a5d062a5612a57a43a171a5b79ddbb690a86a8ddda02339cc1675131ae9f8b" + }, + "ghcr.io/github/gh-aw-firewall/squid:0.25.55": { + "image": "ghcr.io/github/gh-aw-firewall/squid:0.25.55", + "digest": "sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca", + "pinned_image": "ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca" + }, + "ghcr.io/github/gh-aw-mcpg:v0.1.4": { + "image": "ghcr.io/github/gh-aw-mcpg:v0.1.4", + "digest": "sha256:0acf25aa1d409f9c73be9e39ac84f4bd4b90d8bfa1db4dc6d7f47d38ccd58914", + "pinned_image": "ghcr.io/github/gh-aw-mcpg:v0.1.4@sha256:0acf25aa1d409f9c73be9e39ac84f4bd4b90d8bfa1db4dc6d7f47d38ccd58914" + }, + "ghcr.io/github/gh-aw-mcpg:v0.3.19": { + "image": "ghcr.io/github/gh-aw-mcpg:v0.3.19", + "digest": "sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f", + "pinned_image": "ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f" + }, + "ghcr.io/github/github-mcp-server:v0.30.3": { + "image": "ghcr.io/github/github-mcp-server:v0.30.3", + "digest": "sha256:a2b5fb79b1cee851bfc3532dfe480c3dc5736974ca9d93a7a9f68e52ce4b62a0", + "pinned_image": "ghcr.io/github/github-mcp-server:v0.30.3@sha256:a2b5fb79b1cee851bfc3532dfe480c3dc5736974ca9d93a7a9f68e52ce4b62a0" + } + } +} diff --git a/.github/aw/imports/github/gh-aw/359795d49ada21681ab616bd4cbcb144a7387115/.github_workflows_shared_reporting.md b/.github/aw/imports/github/gh-aw/359795d49ada21681ab616bd4cbcb144a7387115/.github_workflows_shared_reporting.md new file mode 100644 index 000000000000..72d61b434efa --- /dev/null +++ b/.github/aw/imports/github/gh-aw/359795d49ada21681ab616bd4cbcb144a7387115/.github_workflows_shared_reporting.md @@ -0,0 +1,73 @@ +--- +# Report formatting guidelines +--- + +## Report Structure Guidelines + +### 1. Header Levels +**Use h3 (###) or lower for all headers in your issue report to maintain proper document hierarchy.** + +When creating GitHub issues or discussions: +- Use `###` (h3) for main sections (e.g., "### Test Summary") +- Use `####` (h4) for subsections (e.g., "#### Device-Specific Results") +- Never use `##` (h2) or `#` (h1) in reports - these are reserved for titles + +### 2. Progressive Disclosure +**Wrap detailed test results in `
Section Name` tags to improve readability and reduce scrolling.** + +Use collapsible sections for: +- Verbose details (full test logs, raw data) +- Secondary information (minor warnings, extra context) +- Per-item breakdowns when there are many items + +Always keep critical information visible (summary, critical issues, key metrics). + +### 3. Report Structure Pattern + +1. **Overview**: 1-2 paragraphs summarizing key findings +2. **Critical Information**: Show immediately (summary stats, critical issues) +3. **Details**: Use `
Section Name` for expanded content +4. **Context**: Add helpful metadata (workflow run, date, trigger) + +### Design Principles (Airbnb-Inspired) + +Reports should: +- **Build trust through clarity**: Most important info immediately visible +- **Exceed expectations**: Add helpful context like trends, comparisons +- **Create delight**: Use progressive disclosure to reduce overwhelm +- **Maintain consistency**: Follow patterns across all reports + +### Example Report Structure + +```markdown +### Summary +- Key metric 1: value +- Key metric 2: value +- Status: ✅/⚠️/❌ + +### Critical Issues +[Always visible - these are important] + +
+View Detailed Results + +[Comprehensive details, logs, traces] + +
+ +
+View All Warnings + +[Minor issues and potential problems] + +
+ +### Recommendations +[Actionable next steps - keep visible] +``` + +## Workflow Run References + +- Format run IDs as links: `[§12345](https://github.com/owner/repo/actions/runs/12345)` +- Include up to 3 most relevant run URLs at end under `**References:**` +- Do NOT add footer attribution (system adds automatically) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index cef74aafb4b0..6ffb926f6fa0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -31,6 +31,8 @@ updates: github-actions-dependencies: patterns: - "*" + ignore: + - dependency-name: "github/gh-aw-actions/**" # Managed by gh aw compile. Version-locked to the gh-aw compiler; do not bump. cooldown: default-days: 7 - package-ecosystem: "maven" diff --git a/.github/workflows/daily-repo-status.lock.yml b/.github/workflows/daily-repo-status.lock.yml index c9b85fafafdd..7c3d20a166ea 100644 --- a/.github/workflows/daily-repo-status.lock.yml +++ b/.github/workflows/daily-repo-status.lock.yml @@ -1,393 +1,598 @@ -# -# ___ _ +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"ae44897dc32d20c421588b31a279653abf436964ad86c58885e0edbfbb2f0416","compiler_version":"v0.76.1","strict":true,"agent_id":"copilot"} +# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"46d564922b082d0db93244972e8005ea6904ee5f","version":"v0.76.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.55","digest":"sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55","digest":"sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.55","digest":"sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.19","digest":"sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f"},{"image":"ghcr.io/github/github-mcp-server:v1.0.4","digest":"sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4","pinned_image":"ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4"},{"image":"node:lts-alpine","digest":"sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14","pinned_image":"node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14"}]} +# ___ _ _ # / _ \ | | (_) -# | |_| | __ _ ___ _ __ | |_ _ +# | |_| | __ _ ___ _ __ | |_ _ ___ # | _ |/ _` |/ _ \ '_ \| __| |/ __| -# | | | | (_| | __/ | | | |_| | ( +# | | | | (_| | __/ | | | |_| | (__ # \_| |_/\__, |\___|_| |_|\__|_|\___| # __/ | # _ _ |___/ # | | | | / _| | -# | | | | ___ _ __ _ __| |_| | _____ +# | | | | ___ _ __ _ __| |_| | _____ ____ # | |/\| |/ _ \ '__| |/ /| _| |/ _ \ \ /\ / / ___| # \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \ # \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ # -# This file was automatically generated by gh-aw (v0.45.0). DO NOT EDIT. +# This file was automatically generated by gh-aw (v0.76.1). DO NOT EDIT. # -# To update this file, edit githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87 and run: -# gh aw +# To update this file, edit githubnext/agentics/workflows/repo-status.md@main and run: +# gh aw compile # Not all edits will cause changes to this file. # # For more information: https://github.github.com/gh-aw/introduction/overview/ # -# This workflow creates daily repo status reports. It gathers recent -# activity (issues, PRs, discussions, releases, code changes) and +# This workflow creates daily repo status reports. It gathers recent repository +# activity (issues, PRs, discussions, releases, code changes) and generates # engaging GitHub issues with productivity insights, community highlights, # and project recommendations. # -# Source: githubnext/agentics/workflows/daily-repo-status.md@ +# Source: githubnext/agentics/workflows/repo-status.md@main +# +# Secrets used: +# - COPILOT_GITHUB_TOKEN +# - GH_AW_GITHUB_MCP_SERVER_TOKEN +# - GH_AW_GITHUB_TOKEN +# - GITHUB_TOKEN +# +# Custom actions used: +# - actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 +# - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 +# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 +# - actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 +# - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 +# - github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 # -# frontmatter-hash: +# Container images used: +# - ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731 +# - ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3 +# - ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca +# - ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f +# - ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4 +# - node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14 -name: "Daily Repo Status" -"on": +name: "Repo Status" +on: schedule: - - cron: "25 18 * * *" + - cron: "11 19 * * *" # Friendly format: daily (scattered) workflow_dispatch: + inputs: + aw_context: + default: "" + description: "Agent caller context (used internally by Agentic Workflows)." + required: false + type: string permissions: {} concurrency: group: "gh-aw-${{ github.workflow }}" -run-name: "Daily Repo Status" +run-name: "Repo Status" jobs: activation: - runs-on: ubuntu- + runs-on: ubuntu-slim permissions: - contents: + actions: read + contents: read outputs: comment_id: "" comment_repo: "" + engine_id: ${{ steps.generate_aw_info.outputs.engine_id }} + lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }} + model: ${{ steps.generate_aw_info.outputs.model }} + secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} + stale_lock_file_failed: ${{ steps.check-lock-file.outputs.stale_lock_file_failed == 'true' }} steps: - - name: Setup - uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.45. + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Repo Status" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-repo-status.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Generate agentic run info + id: generate_aw_info + env: + GH_AW_INFO_ENGINE_ID: "copilot" + GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" + GH_AW_INFO_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || 'claude-sonnet-4.6' }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AGENT_VERSION: "1.0.52" + GH_AW_INFO_CLI_VERSION: "v0.76.1" + GH_AW_INFO_WORKFLOW_NAME: "Repo Status" + GH_AW_INFO_EXPERIMENTAL: "false" + GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" + GH_AW_INFO_STAGED: "false" + GH_AW_INFO_ALLOWED_DOMAINS: '["defaults"]' + GH_AW_INFO_FIREWALL_ENABLED: "true" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_AWMG_VERSION: "" + GH_AW_INFO_FIREWALL_TYPE: "squid" + GH_AW_INFO_FRONTMATTER_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_COMPILED_STRICT: "true" + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); + await main(core, context); + - name: Validate COPILOT_GITHUB_TOKEN secret + id: validate-secret + run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_multi_secret.sh" COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Checkout .github and .agents folders + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - destination: /opt/gh-aw/ - - name: Check workflow file - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # + persist-credentials: false + sparse-checkout: | + .github + .agents + .antigravity + .claude + .codex + .crush + .gemini + .opencode + .pi + sparse-checkout-cone-mode: true + fetch-depth: 1 + - name: Save agent config folders for base branch restoration + env: + GH_AW_AGENT_FOLDERS: ".agents .antigravity .claude .codex .crush .gemini .github .opencode .pi" + GH_AW_AGENT_FILES: ".crush.json AGENTS.md ANTIGRAVITY.md CLAUDE.md GEMINI.md PI.md opencode.jsonc" + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/save_base_github_folders.sh" + - name: Check workflow lock file + id: check-lock-file + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_WORKFLOW_FILE: "daily-repo-status.lock.yml" + GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}" with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/check_workflow_timestamp_api.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_timestamp_api.cjs'); + await main(); + - name: Check compile-agentic version + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_COMPILED_VERSION: "v0.76.1" + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_version_updates.cjs'); + await main(); + - name: Create prompt with built-in context + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl + GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + # poutine:ignore untrusted_checkout_exec + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" + { + cat << 'GH_AW_PROMPT_603e0a48f86a9470_EOF' + + GH_AW_PROMPT_603e0a48f86a9470_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" + cat << 'GH_AW_PROMPT_603e0a48f86a9470_EOF' + + Tools: create_issue, missing_tool, missing_data, noop + + GH_AW_PROMPT_603e0a48f86a9470_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" + cat << 'GH_AW_PROMPT_603e0a48f86a9470_EOF' + + The following GitHub context information is available for this workflow: + {{#if github.actor}} + - **actor**: __GH_AW_GITHUB_ACTOR__ + {{/if}} + {{#if github.repository}} + - **repository**: __GH_AW_GITHUB_REPOSITORY__ + {{/if}} + {{#if github.workspace}} + - **workspace**: __GH_AW_GITHUB_WORKSPACE__ + {{/if}} + {{#if github.event.issue.number || (github.aw.context.item_type == 'issue' && github.aw.context.item_number)}} + - **issue-number**: #__GH_AW_EXPR_802A9F6A__ + {{/if}} + {{#if github.event.discussion.number || (github.aw.context.item_type == 'discussion' && github.aw.context.item_number)}} + - **discussion-number**: #__GH_AW_EXPR_1A3A194A__ + {{/if}} + {{#if github.event.pull_request.number || (github.aw.context.item_type == 'pull_request' && github.aw.context.item_number)}} + - **pull-request-number**: #__GH_AW_EXPR_463A214A__ + {{/if}} + {{#if github.event.comment.id || github.aw.context.comment_id}} + - **comment-id**: __GH_AW_EXPR_FF1D34CE__ + {{/if}} + {{#if github.run_id}} + - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ + {{/if}} + + + GH_AW_PROMPT_603e0a48f86a9470_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" + cat << 'GH_AW_PROMPT_603e0a48f86a9470_EOF' + + {{#runtime-import .github/workflows/daily-repo-status.md}} + GH_AW_PROMPT_603e0a48f86a9470_EOF + } > "$GH_AW_PROMPT" + - name: Interpolate variables and render templates + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_ENGINE_ID: "copilot" + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/interpolate_prompt.cjs'); await main(); + - name: Substitute placeholders + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + GH_AW_MCP_CLI_SERVERS_LIST: '- `safeoutputs` — run `safeoutputs --help` to see available tools' + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + + const substitutePlaceholders = require('${{ runner.temp }}/gh-aw/actions/substitute_placeholders.cjs'); + + // Call the substitution function + return await substitutePlaceholders({ + file: process.env.GH_AW_PROMPT, + substitutions: { + GH_AW_EXPR_1A3A194A: process.env.GH_AW_EXPR_1A3A194A, + GH_AW_EXPR_463A214A: process.env.GH_AW_EXPR_463A214A, + GH_AW_EXPR_802A9F6A: process.env.GH_AW_EXPR_802A9F6A, + GH_AW_EXPR_FF1D34CE: process.env.GH_AW_EXPR_FF1D34CE, + GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, + GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, + GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, + GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, + GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST + } + }); + - name: Validate prompt placeholders + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh" + - name: Print prompt + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh" + - name: Upload activation artifact + if: success() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: activation + include-hidden-files: true + path: | + /tmp/gh-aw/aw_info.json + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/aw-prompts/prompt-template.txt + /tmp/gh-aw/aw-prompts/prompt-import-tree.json + /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/base + /tmp/gh-aw/.github/agents + /tmp/gh-aw/.github/skills + if-no-files-found: ignore + retention-days: 1 agent: - needs: - runs-on: ubuntu- + needs: activation + runs-on: ubuntu-latest permissions: - contents: - issues: - pull-requests: + contents: read + issues: read + pull-requests: read concurrency: group: "gh-aw-copilot-${{ github.workflow }}" env: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} GH_AW_ASSETS_ALLOWED_EXTS: "" GH_AW_ASSETS_BRANCH: "" - GH_AW_ASSETS_MAX_SIZE_KB: - GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/ - GH_AW_SAFE_OUTPUTS: /opt/gh-aw/safeoutputs/outputs. - GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config. - GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools. - GH_AW_WORKFLOW_ID_SANITIZED: + GH_AW_ASSETS_MAX_SIZE_KB: 0 + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + GH_AW_WORKFLOW_ID_SANITIZED: dailyrepostatus outputs: + agentic_engine_timeout: ${{ steps.detect-agent-errors.outputs.agentic_engine_timeout || 'false' }} checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} + effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }} + effective_tokens_rate_limit_error: ${{ steps.parse-mcp-gateway.outputs.effective_tokens_rate_limit_error || 'false' }} has_patch: ${{ steps.collect_output.outputs.has_patch }} - model: ${{ steps.generate_aw_info.outputs.model }} + inference_access_error: ${{ steps.detect-agent-errors.outputs.inference_access_error || 'false' }} + mcp_policy_error: ${{ steps.detect-agent-errors.outputs.mcp_policy_error || 'false' }} + model: ${{ needs.activation.outputs.model }} + model_not_supported_error: ${{ steps.detect-agent-errors.outputs.model_not_supported_error || 'false' }} output: ${{ steps.collect_output.outputs.output }} output_types: ${{ steps.collect_output.outputs.output_types }} - secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} steps: - - name: Setup - uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.45. + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/ - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0. + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Repo Status" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-repo-status.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Set runtime paths + id: set-runtime-paths + run: | + { + echo "GH_AW_SAFE_OUTPUTS=${RUNNER_TEMP}/gh-aw/safeoutputs/outputs.jsonl" + echo "GH_AW_SAFE_OUTPUTS_CONFIG_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" + echo "GH_AW_SAFE_OUTPUTS_TOOLS_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/tools.json" + } >> "$GITHUB_OUTPUT" + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - name: Create gh-aw temp - run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir. - - name: Configure Git + - name: Create gh-aw temp directory + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_gh_aw_tmp_dir.sh" + - name: Configure gh CLI for GitHub Enterprise + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" + env: + GH_TOKEN: ${{ github.token }} + - name: Configure Git credentials env: REPO_NAME: ${{ github.repository }} SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} run: | git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" - # Re-authenticate git with GitHub + git config --global am.keepcr true + # Re-authenticate git with GitHub token SERVER_URL_STRIPPED="${SERVER_URL#https://}" - git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" echo "Git configured with standard GitHub Actions identity" - - name: Checkout PR - id: checkout- + - name: Checkout PR branch + id: checkout-pr if: | - github.event. - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # + github.event.pull_request || github.event.issue.pull_request + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} with: github-token: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/checkout_pr_branch.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/checkout_pr_branch.cjs'); await main(); - - name: Generate agentic run - id: - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # + - name: Install GitHub Copilot CLI + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.52 + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.25.55 + - name: Parse integrity filter lists + id: parse-guard-vars + env: + GH_AW_BLOCKED_USERS_VAR: ${{ vars.GH_AW_GITHUB_BLOCKED_USERS || '' }} + GH_AW_TRUSTED_USERS_VAR: ${{ vars.GH_AW_GITHUB_TRUSTED_USERS || '' }} + GH_AW_APPROVAL_LABELS_VAR: ${{ vars.GH_AW_GITHUB_APPROVAL_LABELS || '' }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/parse_guard_list.sh" + - name: Download activation artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - script: | - const fs = require('fs'); - - const awInfo = { - engine_id: "copilot", - engine_name: "GitHub Copilot CLI", - model: process.env.GH_AW_MODEL_AGENT_COPILOT || "", - version: "", - agent_version: "0.0.410", - cli_version: "v0.45.0", - workflow_name: "Daily Repo Status", - experimental: false, - supports_tools_allowlist: true, - supports_http_transport: true, - run_id: context.runId, - run_number: context.runNumber, - run_attempt: process.env.GITHUB_RUN_ATTEMPT, - repository: context.repo.owner + '/' + context.repo.repo, - ref: context.ref, - sha: context.sha, - actor: context.actor, - event_name: context.eventName, - staged: false, - allowed_domains: ["defaults"], - firewall_enabled: true, - awf_version: "v0.18.0", - awmg_version: "v0.1.4", - steps: { - firewall: "squid" - }, - created_at: new Date().toISOString() - }; - - // Write to /tmp/gh-aw directory to avoid inclusion in - const tmpPath = '/tmp/gh-aw/aw_info.json'; - fs.writeFileSync(tmpPath, JSON.stringify(awInfo, null, 2)); - console.log('Generated aw_info.json at:', tmpPath); - console.log(JSON.stringify(awInfo, null, 2)); - - // Set model as output for reuse in other steps/ - core.setOutput('model', awInfo.model); - - name: Validate COPILOT_GITHUB_TOKEN - id: validate- - run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot- + name: activation + path: /tmp/gh-aw + - name: Restore agent config folders from base branch + if: steps.checkout-pr.outcome == 'success' env: - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - - name: Install GitHub Copilot - run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0. - - name: Install awf - run: bash /opt/gh-aw/actions/install_awf_binary.sh v0.18. - - name: Download container - run: bash /opt/gh-aw/actions/download_docker_images.sh ghcr.io/github/gh-aw-firewall/agent:0.18.0 ghcr.io/github/gh-aw-firewall/squid:0.18.0 ghcr.io/github/gh-aw-mcpg:v0.1.4 ghcr.io/github/github-mcp-server:v0.30.3 node:lts- - - name: Write Safe Outputs + GH_AW_AGENT_FOLDERS: ".agents .antigravity .claude .codex .crush .gemini .github .opencode .pi" + GH_AW_AGENT_FILES: ".crush.json AGENTS.md ANTIGRAVITY.md CLAUDE.md GEMINI.md PI.md opencode.jsonc" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_base_github_folders.sh" + - name: Restore inline sub-agents from activation artifact + env: + GH_AW_SUB_AGENT_DIR: ".github/agents" + GH_AW_SUB_AGENT_EXT: ".agent.md" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_sub_agents.sh" + - name: Restore inline skills from activation artifact + env: + GH_AW_SKILL_DIR: ".github/skills" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_skills.sh" + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3 ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4 node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14 + - name: Generate Safe Outputs Config run: | - mkdir -p /opt/gh-aw/ - mkdir -p /tmp/gh-aw/ - mkdir -p /tmp/gh-aw/mcp-logs/ - cat > /opt/gh-aw/safeoutputs/config.json << 'GH_AW_SAFE_OUTPUTS_CONFIG_EOF' - {"create_issue":{"max":1},"missing_data":{},"missing_tool":{},"noop":{"max":1}} - - cat > /opt/gh-aw/safeoutputs/tools.json << 'GH_AW_SAFE_OUTPUTS_TOOLS_EOF' - [ + mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" + mkdir -p /tmp/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_9b6646f0d620a78f_EOF' + {"create_issue":{"close_older_issues":true,"labels":["report","daily-status"],"max":1,"title_prefix":"[repo-status] "},"create_report_incomplete_issue":{},"mentions":{"enabled":false},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} + GH_AW_SAFE_OUTPUTS_CONFIG_9b6646f0d620a78f_EOF + - name: Generate Safe Outputs Tools + env: + GH_AW_TOOLS_META_JSON: | { - "description": "Create a new GitHub issue for tracking bugs, feature requests, or tasks. Use this for actionable work items that need assignment, labeling, and status tracking. For reports, announcements, or status updates that don't require task tracking, use create_discussion instead. CONSTRAINTS: Maximum 1 issue(s) can be created. Title will be prefixed with \"[repo-status] \". Labels [report daily-status] will be automatically added.", - "inputSchema": { - "additionalProperties": false, - "properties": { + "description_suffixes": { + "create_issue": " CONSTRAINTS: Maximum 1 issue(s) can be created. Title will be prefixed with \"[repo-status] \". Labels [\"report\" \"daily-status\"] will be automatically added." + }, + "repo_params": {}, + "dynamic_tools": [] + } + GH_AW_VALIDATION_JSON: | + { + "create_issue": { + "defaultMax": 1, + "fields": { "body": { - "description": "Detailed issue description in Markdown. Do NOT repeat the title as a heading since it already appears as the issue's h1. Include context, reproduction steps, or acceptance criteria as appropriate.", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 }, - "labels": { - "description": "Labels to categorize the issue (e.g., 'bug', 'enhancement'). Labels must exist in the repository.", - "items": { - "type": "string" - }, + "fields": { "type": "array" }, + "labels": { + "type": "array", + "itemType": "string", + "itemSanitize": true, + "itemMaxLength": 128 + }, "parent": { - "description": "Parent issue number for creating sub-issues. This is the numeric ID from the GitHub URL (e.g., 42 in github.com/owner/repo/issues/42). Can also be a temporary_id (e.g., 'aw_abc123', 'aw_Test123') from a previously created issue in the same workflow run.", - "type": [ - "number", - "string" - ] + "issueOrPRNumber": true + }, + "repo": { + "type": "string", + "maxLength": 256 }, "temporary_id": { - "description": "Unique temporary identifier for referencing this issue before it's created. Format: 'aw_' followed by 3 to 8 alphanumeric characters (e.g., 'aw_abc1', 'aw_Test123'). Use '#aw_ID' in body text to reference other issues by their temporary_id; these are replaced with actual issue numbers after creation.", - "pattern": "^aw_[A-Za-z0-9]{4,8}$", "type": "string" }, "title": { - "description": "Concise issue title summarizing the bug, feature, or task. The title appears as the main heading, so keep it brief and descriptive.", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 128 } - }, - "required": [ - "title", - "body" - ], - "type": "object" + } }, - "name": "create_issue" - }, - { - "description": "Report that a tool or capability needed to complete the task is not available, or share any information you deem important about missing functionality or limitations. Use this when you cannot accomplish what was requested because the required functionality is missing or access is restricted.", - "inputSchema": { - "additionalProperties": false, - "properties": { + "missing_data": { + "defaultMax": 20, + "fields": { "alternatives": { - "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", - "type": "string" + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "context": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "data_type": { + "type": "string", + "sanitize": true, + "maxLength": 128 }, "reason": { - "description": "Explanation of why this tool is needed or what information you want to share about the limitation (max 256 characters).", - "type": "string" + "type": "string", + "sanitize": true, + "maxLength": 256 + } + } + }, + "missing_tool": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 512 + }, + "reason": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 }, "tool": { - "description": "Optional: Name or description of the missing tool or capability (max 128 characters). Be specific about what functionality is needed.", - "type": "string" + "type": "string", + "sanitize": true, + "maxLength": 128 } - }, - "required": [ - "reason" - ], - "type": "object" + } }, - "name": "missing_tool" - }, - { - "description": "Log a transparency message when no significant actions are needed. Use this to confirm workflow completion and provide visibility when analysis is complete but no changes or outputs are required (e.g., 'No issues found', 'All checks passed'). This ensures the workflow produces human-visible output even when no other actions are taken.", - "inputSchema": { - "additionalProperties": false, - "properties": { + "noop": { + "defaultMax": 1, + "fields": { "message": { - "description": "Status or completion message to log. Should explain what was analyzed and the outcome (e.g., 'Code review complete - no issues found', 'Analysis complete - all tests passing').", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 } - }, - "required": [ - "message" - ], - "type": "object" + } }, - "name": "noop" - }, - { - "description": "Report that data or information needed to complete the task is not available. Use this when you cannot accomplish what was requested because required data, context, or information is missing.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "alternatives": { - "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", - "type": "string" - }, - "context": { - "description": "Additional context about the missing data or where it should come from (max 256 characters).", - "type": "string" - }, - "data_type": { - "description": "Type or description of the missing data or information (max 128 characters). Be specific about what data is needed.", - "type": "string" + "report_incomplete": { + "defaultMax": 5, + "fields": { + "details": { + "type": "string", + "sanitize": true, + "maxLength": 65000 }, "reason": { - "description": "Explanation of why this data is needed to complete the task (max 256 characters).", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 1024 } - }, - "required": [], - "type": "object" - }, - "name": "missing_data" - } - ] - - cat > /opt/gh-aw/safeoutputs/validation.json << 'GH_AW_SAFE_OUTPUTS_VALIDATION_EOF' - { - "create_issue": { - "defaultMax": 1, - "fields": { - "body": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": - }, - "labels": { - "type": "array", - "itemType": "string", - "itemSanitize": true, - "itemMaxLength": - }, - "parent": { - "issueOrPRNumber": - }, - "repo": { - "type": "string", - "maxLength": - }, - "temporary_id": { - "type": "string" - }, - "title": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": - } - } - }, - "missing_tool": { - "defaultMax": 20, - "fields": { - "alternatives": { - "type": "string", - "sanitize": true, - "maxLength": - }, - "reason": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": - }, - "tool": { - "type": "string", - "sanitize": true, - "maxLength": - } - } - }, - "noop": { - "defaultMax": 1, - "fields": { - "message": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": } } } - } - - - name: Generate Safe Outputs MCP Server - id: safe-outputs- + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_safe_outputs_tools.cjs'); + await main(); + - name: Generate Safe Outputs MCP Server Config + id: safe-outputs-config run: | # Generate a secure random API key (360 bits of entropy, 40+ chars) - # Mask immediately to prevent timing + # Mask immediately to prevent timing vulnerabilities API_KEY=$(openssl rand -base64 45 | tr -d '/+=') echo "::add-mask::${API_KEY}" - PORT= + PORT=3001 - # Set outputs for next + # Set outputs for next steps { echo "safe_outputs_api_key=${API_KEY}" echo "safe_outputs_port=${PORT}" @@ -395,61 +600,84 @@ jobs: echo "Safe Outputs MCP server will run on port ${PORT}" - - name: Start Safe Outputs MCP HTTP - id: safe-outputs- + - name: Start Safe Outputs MCP HTTP Server + id: safe-outputs-start env: DEBUG: '*' + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-config.outputs.safe_outputs_port }} GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-config.outputs.safe_outputs_api_key }} - GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools. - GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config. - GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/ + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: ${{ runner.temp }}/gh-aw/safeoutputs/tools.json + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: ${{ runner.temp }}/gh-aw/safeoutputs/config.json + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs run: | - # Environment variables are set above to prevent template - export - export - export - export - export - export + # Environment variables are set above to prevent template injection + export DEBUG + export GH_AW_SAFE_OUTPUTS + export GH_AW_SAFE_OUTPUTS_PORT + export GH_AW_SAFE_OUTPUTS_API_KEY + export GH_AW_SAFE_OUTPUTS_TOOLS_PATH + export GH_AW_SAFE_OUTPUTS_CONFIG_PATH + export GH_AW_MCP_LOG_DIR - bash /opt/gh-aw/actions/start_safe_outputs_server. + bash "${RUNNER_TEMP}/gh-aw/actions/start_safe_outputs_server.sh" - - name: Start MCP - id: start-mcp- + - name: Start MCP Gateway + id: start-mcp-gateway env: - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-start.outputs.api_key }} GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-start.outputs.port }} GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} run: | - set -eo - mkdir -p /tmp/gh-aw/mcp- + set -eo pipefail + mkdir -p "${RUNNER_TEMP}/gh-aw/mcp-config" - # Export gateway environment variables for MCP config and gateway - export MCP_GATEWAY_PORT="80" + # Export gateway environment variables for MCP config and gateway script + export MCP_GATEWAY_PORT="8080" export MCP_GATEWAY_DOMAIN="host.docker.internal" + export MCP_GATEWAY_HOST_DOMAIN="localhost" MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') echo "::add-mask::${MCP_GATEWAY_API_KEY}" - export + export MCP_GATEWAY_API_KEY export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" + export MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD="524288" export DEBUG="*" export GH_AW_ENGINE="copilot" - export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_LOCKDOWN -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.1.4' + MCP_GATEWAY_UID=$(id -u 2>/dev/null || echo '0') + MCP_GATEWAY_GID=$(id -g 2>/dev/null || echo '0') + case "${DOCKER_HOST:-}" in + unix://* ) DOCKER_SOCK_PATH="${DOCKER_HOST#unix://}" ;; + /* ) DOCKER_SOCK_PATH="$DOCKER_HOST" ;; + * ) DOCKER_SOCK_PATH=/var/run/docker.sock ;; + esac + DOCKER_SOCK_GID=$(stat -c '%g' "$DOCKER_SOCK_PATH" 2>/dev/null || echo '0') + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host --add-host host.docker.internal:127.0.0.1 --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.3.19' - mkdir -p /home/runner/. - cat << GH_AW_MCP_CONFIG_EOF | bash /opt/gh-aw/actions/start_mcp_gateway. + mkdir -p /home/runner/.copilot + GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) + cat << GH_AW_MCP_CONFIG_d075a7f45ab51044_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "github": { "type": "stdio", - "container": "ghcr.io/github/github-mcp-server:v0.30.3", + "container": "ghcr.io/github/github-mcp-server:v1.0.4", "env": { + "GITHUB_HOST": "\${GITHUB_SERVER_URL}", "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", "GITHUB_READ_ONLY": "1", "GITHUB_TOOLSETS": "context,repos,issues,pull_requests" + }, + "guard-policies": { + "allow-only": { + "approval-labels": ${{ steps.parse-guard-vars.outputs.approval_labels }}, + "blocked-users": ${{ steps.parse-guard-vars.outputs.blocked_users }}, + "min-integrity": "none", + "repos": "all", + "trusted-users": ${{ steps.parse-guard-vars.outputs.trusted_users }} + } } }, "safeoutputs": { @@ -457,6 +685,13 @@ jobs: "url": "http://host.docker.internal:$GH_AW_SAFE_OUTPUTS_PORT", "headers": { "Authorization": "\${GH_AW_SAFE_OUTPUTS_API_KEY}" + }, + "guard-policies": { + "write-sink": { + "accept": [ + "*" + ] + } } } }, @@ -467,212 +702,113 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - - - name: Generate workflow - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # - with: - script: | - const { generateWorkflowOverview } = require('/opt/gh-aw/actions/generate_workflow_overview.cjs'); - await generateWorkflowOverview(core); - - name: Create prompt with built-in - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} - GH_AW_GITHUB_ACTOR: ${{ github.actor }} - GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - run: | - bash /opt/gh-aw/actions/create_prompt_first. - cat << 'GH_AW_PROMPT_EOF' > "$GH_AW_PROMPT" - - - cat "/opt/gh-aw/prompts/xpia.md" >> "$GH_AW_PROMPT" - cat "/opt/gh-aw/prompts/temp_folder_prompt.md" >> "$GH_AW_PROMPT" - cat "/opt/gh-aw/prompts/markdown.md" >> "$GH_AW_PROMPT" - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - - GitHub API Access Instructions - - The gh CLI is NOT authenticated. Do NOT use gh commands for GitHub operations. - - - To create or modify GitHub resources (issues, discussions, pull requests, etc.), you MUST call the appropriate safe output tool. Simply writing content will NOT work - the workflow requires actual tool calls. - - Temporary IDs: Some safe output tools support a temporary ID field (usually named temporary_id) so you can reference newly-created items elsewhere in the SAME agent output (for example, using #aw_abc1 in a later body). - - **IMPORTANT - temporary_id format rules:** - - If you DON'T need to reference the item later, OMIT the temporary_id field entirely (it will be auto-generated if needed) - - If you DO need cross-references/chaining, you MUST match this EXACT validation regex: /^aw_[A-Za-z0-9]{3,8}$/ - - Format: aw_ prefix followed by 3 to 8 alphanumeric characters (A-Z, a-z, 0-9, case-insensitive) - - Valid alphanumeric characters: - - INVALID examples: aw_ab (too short), aw_123456789 (too long), aw_test-id (contains hyphen), aw_id_123 (contains underscore) - - VALID examples: aw_abc, aw_abc1, aw_Test123, aw_A1B2C3D4, - - To generate valid IDs: use 3-8 random alphanumeric characters or omit the field to let the system auto- - - Do NOT invent other aw_* formats — downstream steps will reject them with validation errors matching against /^aw_[A-Za-z0-9]{3,8}$/i. - - Discover available tools from the safeoutputs MCP server. - - **Critical**: Tool calls write structured data that downstream jobs process. Without tool calls, follow-up actions will be skipped. - - **Note**: If you made no other safe output tool calls during this workflow execution, call the "noop" tool to provide a status message indicating completion or that no actions were needed. - - - - The following GitHub context information is available for this workflow: - {{#if __GH_AW_GITHUB_ACTOR__ }} - - **actor**: - {{/if}} - {{#if __GH_AW_GITHUB_REPOSITORY__ }} - - **repository**: - {{/if}} - {{#if __GH_AW_GITHUB_WORKSPACE__ }} - - **workspace**: - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ }} - - **issue-number**: # - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ }} - - **discussion-number**: # - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ }} - - **pull-request-number**: # - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_COMMENT_ID__ }} - - **comment-id**: - {{/if}} - {{#if __GH_AW_GITHUB_RUN_ID__ }} - - **workflow-run-id**: - {{/if}} - - - - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - - - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - {{#runtime-import .github/workflows/daily-repo-status.md}} - - - name: Substitute - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. - GH_AW_GITHUB_ACTOR: ${{ github.actor }} - GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - with: - script: | - const substitutePlaceholders = require('/opt/gh-aw/actions/substitute_placeholders.cjs'); - - // Call the substitution - return await substitutePlaceholders({ - file: process.env.GH_AW_PROMPT, - substitutions: { - GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, - GH_AW_GITHUB_EVENT_COMMENT_ID: process.env.GH_AW_GITHUB_EVENT_COMMENT_ID, - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: process.env.GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER, - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: process.env.GH_AW_GITHUB_EVENT_ISSUE_NUMBER, - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: process.env.GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER, - GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, - GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, - GH_AW_GITHUB_WORKSPACE: process.env. - } - }); - - name: Interpolate variables and render - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # + GH_AW_MCP_CONFIG_d075a7f45ab51044_EOF + - name: Mount MCP servers as CLIs + id: mount-mcp-clis + continue-on-error: true env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. + MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + MCP_GATEWAY_DOMAIN: ${{ steps.start-mcp-gateway.outputs.gateway-domain }} + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/interpolate_prompt.cjs'); + const { main } = require('${{ runner.temp }}/gh-aw/actions/mount_mcp_as_cli.cjs'); await main(); - - name: Validate prompt - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. - run: bash /opt/gh-aw/actions/validate_prompt_placeholders. - - name: Print - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. - run: bash /opt/gh-aw/actions/print_prompt_summary. - - name: Clean git - run: bash /opt/gh-aw/actions/clean_git_credentials. - - name: Execute GitHub Copilot - id: + - name: Clean credentials + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/clean_git_credentials.sh" + - name: Audit pre-agent workspace + id: pre_agent_audit + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/audit_pre_agent_workspace.sh" + - name: Execute GitHub Copilot CLI + id: agentic_execution # Copilot CLI tool arguments (sorted): - timeout-minutes: + timeout-minutes: 20 run: | - set -o - sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.18.0 --skip-pull \ - -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --add-dir "${GITHUB_WORKSPACE}" --disable-builtin-mcps --allow-all-tools --allow-all-paths --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"${GH_AW_MODEL_AGENT_COPILOT:+ --model "$GH_AW_MODEL_AGENT_COPILOT"}' 2>&1 | tee -a /tmp/gh-aw/agent-stdio. + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/agent-stdio.log) + printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.25.55/awf-config.schema.json","network":{"allowDomains":["api.business.githubcopilot.com","api.enterprise.githubcopilot.com","api.github.com","api.githubcopilot.com","api.individual.githubcopilot.com","api.snapcraft.io","archive.ubuntu.com","azure.archive.ubuntu.com","crl.geotrust.com","crl.globalsign.com","crl.identrust.com","crl.sectigo.com","crl.thawte.com","crl.usertrust.com","crl.verisign.com","crl3.digicert.com","crl4.digicert.com","crls.ssl.com","github.com","host.docker.internal","json-schema.org","json.schemastore.org","keyserver.ubuntu.com","ocsp.digicert.com","ocsp.geotrust.com","ocsp.globalsign.com","ocsp.identrust.com","ocsp.sectigo.com","ocsp.ssl.com","ocsp.thawte.com","ocsp.usertrust.com","ocsp.verisign.com","packagecloud.io","packages.cloud.google.com","packages.microsoft.com","ppa.launchpad.net","raw.githubusercontent.com","registry.npmjs.org","s.symcb.com","s.symcd.com","security.ubuntu.com","telemetry.enterprise.githubcopilot.com","ts-crl.ws.symantec.com","ts-ocsp.ws.symantec.com","www.googleapis.com"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxEffectiveTokens":25000000,"models":{"agent":["sonnet-6x","gpt-5.4","gpt-5.3","gemini-pro","any"],"antigravity":["copilot/antigravity*","google/antigravity*","gemini/antigravity*"],"any":["copilot/*","anthropic/*","openai/*","google/*","gemini/*"],"claude":["agent"],"codex":["agent"],"coding":["copilot/gpt-5*codex*","openai/gpt-5*codex*","gpt-5-codex"],"computer-use":["copilot/*computer-use*","google/*computer-use*","gemini/*computer-use*","openai/*computer-use*"],"copilot":["agent"],"deep-research":["copilot/deep-research*","copilot/o3-deep-research*","copilot/o4-mini-deep-research*","google/deep-research*","gemini/deep-research*","openai/o3-deep-research*","openai/o4-mini-deep-research*"],"gemini":["agent"],"gemini-3-flash":["copilot/gemini-3*flash*","google/gemini-3*flash*","gemini/gemini-3*flash*"],"gemini-3-pro":["copilot/gemini-3*pro*","google/gemini-3*pro*","gemini/gemini-3*pro*"],"gemini-3.1-flash":["copilot/gemini-3.1*flash*","google/gemini-3.1*flash*","gemini/gemini-3.1*flash*"],"gemini-3.1-pro":["copilot/gemini-3.1*pro*","google/gemini-3.1*pro*","gemini/gemini-3.1*pro*"],"gemini-3.5-flash":["copilot/gemini-3.5*flash*","google/gemini-3.5*flash*","gemini/gemini-3.5*flash*"],"gemini-flash":["copilot/gemini-*flash*","google/gemini-*flash*","gemini/gemini-*flash*"],"gemini-flash-lite":["copilot/gemini-*flash*lite*","google/gemini-*flash*lite*","gemini/gemini-*flash*lite*"],"gemini-pro":["copilot/gemini-*pro*","google/gemini-*pro*","gemini/gemini-*pro*"],"gemma":["copilot/gemma*","google/gemma*","gemini/gemma*"],"gpt-4.1":["copilot/gpt-4.1*","openai/gpt-4.1*"],"gpt-5":["copilot/gpt-5*","openai/gpt-5*"],"gpt-5-codex":["copilot/gpt-5*codex*","openai/gpt-5*codex*"],"gpt-5-mini":["copilot/gpt-5*mini*","openai/gpt-5*mini*"],"gpt-5-nano":["copilot/gpt-5*nano*","openai/gpt-5*nano*"],"gpt-5-pro":["copilot/gpt-5*pro*","openai/gpt-5*pro*"],"gpt-5.2":["copilot/gpt-5.2*","openai/gpt-5.2*"],"gpt-5.3":["copilot/gpt-5.3*","openai/gpt-5.3*"],"gpt-5.4":["copilot/gpt-5.4*","openai/gpt-5.4*"],"gpt-5.5":["copilot/gpt-5.5*","openai/gpt-5.5*"],"haiku":["copilot/*haiku*","anthropic/*haiku*"],"large":["sonnet","gpt-5-pro","gpt-5","gemini-pro"],"mini":["haiku","gpt-5-mini","gpt-5-nano","gemini-flash-lite"],"opus":["copilot/*opus*","anthropic/*opus*"],"opusplan":["opus?effort=high"],"reasoning":["copilot/o1*","copilot/o3*","copilot/o4*","openai/o1*","openai/o3*","openai/o4*"],"robotics":["copilot/*robotics*","google/*robotics*","gemini/*robotics*"],"small":["mini"],"sonnet":["copilot/*sonnet*","anthropic/*sonnet*"],"sonnet-6x":["copilot/*sonnet-4-5-*","anthropic/*sonnet-4-5-*","copilot/*sonnet-4-6*","anthropic/*sonnet-4-6*"],"summarization":["haiku","gpt-5-mini","gemini-flash-lite","mini"],"vision":["copilot/gemini-*image*","gemini/gemini-*image*","copilot/gemini-*flash*","gemini/gemini-*flash*"]}},"container":{"imageTag":"0.25.55,squid=sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca,agent=sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731,api-proxy=sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw" + fi + # shellcheck disable=SC1003 + sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \ + -- /bin/bash -c 'export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && export PATH="$(find /opt/hostedtoolcache /home/runner/work/_tool -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: - COPILOT_AGENT_RUNNER_TYPE: + AWF_REFLECT_ENABLED: 1 + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config. - GH_AW_MODEL_AGENT_COPILOT: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }} - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || 'claude-sonnet-4.6' }} + GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json + GH_AW_PHASE: agent + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_VERSION: v0.76.1 + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} GITHUB_REF_NAME: ${{ github.ref_name }} - GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md GITHUB_WORKSPACE: ${{ github.workspace }} - XDG_CONFIG_HOME: /home/ - - name: Configure Git + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + XDG_CONFIG_HOME: /home/runner + - name: Detect agent errors + if: always() + id: detect-agent-errors + continue-on-error: true + run: node "${RUNNER_TEMP}/gh-aw/actions/detect_agent_errors.cjs" + - name: Configure Git credentials env: REPO_NAME: ${{ github.repository }} SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} run: | git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" - # Re-authenticate git with GitHub + git config --global am.keepcr true + # Re-authenticate git with GitHub token SERVER_URL_STRIPPED="${SERVER_URL#https://}" - git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" echo "Git configured with standard GitHub Actions identity" - - name: Copy Copilot session state files to + - name: Copy Copilot session state files to logs if: always() - continue-on-error: - run: | - # Copy Copilot session state files to logs folder for artifact - # This ensures they are in /tmp/gh-aw/ where secret redaction can scan - SESSION_STATE_DIR="$HOME/.copilot/session-state" - LOGS_DIR="/tmp/gh-aw/sandbox/agent/logs" - - if [ -d "$SESSION_STATE_DIR" ]; - echo "Copying Copilot session state files from $SESSION_STATE_DIR to $LOGS_DIR" - mkdir -p "$LOGS_DIR" - cp -v "$SESSION_STATE_DIR"/*.jsonl "$LOGS_DIR/" 2>/dev/null || - echo "Session state files copied successfully" - - echo "No session-state directory found at $SESSION_STATE_DIR" - - - name: Stop MCP + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/copy_copilot_session_state.sh" + - name: Stop MCP Gateway if: always() - continue-on-error: + continue-on-error: true env: MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} run: | - bash /opt/gh-aw/actions/stop_mcp_gateway.sh "$GATEWAY_PID" - - name: Redact secrets in + bash "${RUNNER_TEMP}/gh-aw/actions/stop_mcp_gateway.sh" "$GATEWAY_PID" + - name: Redact secrets in logs if: always() - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/redact_secrets.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_secrets.cjs'); await main(); env: GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' @@ -680,343 +816,579 @@ jobs: SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Upload Safe + - name: Append agent step summary if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v6.0. - with: - name: safe- - path: ${{ env.GH_AW_SAFE_OUTPUTS }} - if-no-files-found: - - name: Ingest agent - id: + run: bash "${RUNNER_TEMP}/gh-aw/actions/append_agent_step_summary.sh" + - name: Copy Safe Outputs if: always() - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # env: - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} - GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + run: | + mkdir -p /tmp/gh-aw + cp "$GH_AW_SAFE_OUTPUTS" /tmp/gh-aw/safeoutputs.jsonl 2>/dev/null || true + - name: Ingest agent output + id: collect_output + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GH_AW_ALLOWED_GITHUB_REFS: "" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/collect_ndjson_output.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/collect_ndjson_output.cjs'); await main(); - - name: Upload sanitized agent - if: always() && env. - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v6.0. - with: - name: agent- - path: ${{ env.GH_AW_AGENT_OUTPUT }} - if-no-files-found: - - name: Upload engine output - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v6.0. - with: - name: - path: | - /tmp/gh-aw/sandbox/agent/logs/ - /tmp/gh-aw/redacted-urls. - if-no-files-found: - - name: Parse agent logs for step + - name: Parse agent logs for step summary if: always() - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/parse_copilot_log.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_copilot_log.cjs'); await main(); - - name: Parse MCP Gateway logs for step + - name: Parse MCP Gateway logs for step summary if: always() - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/parse_mcp_gateway_log.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); await main(); - - name: Print firewall + - name: Print firewall logs if: always() - continue-on-error: + continue-on-error: true env: - AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/ + AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs run: | - # Fix permissions on firewall logs so they can be uploaded as - # AWF runs with sudo, creating files owned by - sudo chmod -R a+r /tmp/gh-aw/sandbox/firewall/logs 2>/dev/null || + # Fix permissions on firewall logs/audit dirs so they can be uploaded as artifacts + # AWF runs with sudo, creating files owned by root + sudo chmod -R a+rX /tmp/gh-aw/sandbox/firewall 2>/dev/null || true # Only run awf logs summary if awf command exists (it may not be installed if workflow failed before install step) - if command -v awf &> /dev/null; + if command -v awf &> /dev/null; then awf logs summary | tee -a "$GITHUB_STEP_SUMMARY" - + else echo 'AWF binary not installed, skipping firewall log summary' - - - name: Upload agent + fi + - name: Parse token usage for step summary + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_token_usage.cjs'); + await main(); + - name: Print AWF reflect summary + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/awf_reflect_summary.cjs'); + await main(); + - name: Write agent output placeholder if missing if: always() - continue-on-error: - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v6.0. + run: | + if [ ! -f /tmp/gh-aw/agent_output.json ]; then + echo '{"items":[]}' > /tmp/gh-aw/agent_output.json + fi + - name: Upload agent artifacts + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: agent- + name: agent path: | - /tmp/gh-aw/aw-prompts/prompt. - /tmp/gh-aw/aw_info. + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/sandbox/agent/logs/ + /tmp/gh-aw/redacted-urls.log /tmp/gh-aw/mcp-logs/ - /tmp/gh-aw/sandbox/firewall/logs/ - /tmp/gh-aw/agent-stdio. + /tmp/gh-aw/proxy-logs/ + !/tmp/gh-aw/proxy-logs/proxy-tls/ + /tmp/gh-aw/agent_usage.json + /tmp/gh-aw/agent-stdio.log + /tmp/gh-aw/pre-agent-audit.txt /tmp/gh-aw/agent/ - if-no-files-found: + /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/safeoutputs.jsonl + /tmp/gh-aw/agent_output.json + /tmp/gh-aw/aw-*.patch + /tmp/gh-aw/aw-*.bundle + /tmp/gh-aw/awf-config.json + /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/sandbox/firewall/audit/ + /tmp/gh-aw/sandbox/firewall/awf-reflect.json + if-no-files-found: ignore conclusion: needs: - - - - - - - - - if: (always()) && (needs.agent.result != 'skipped') - runs-on: ubuntu- + - activation + - agent + - detection + - safe_outputs + if: > + always() && (needs.agent.result != 'skipped' || needs.activation.outputs.lockdown_check_failed == 'true' || + needs.activation.outputs.stale_lock_file_failed == 'true') + runs-on: ubuntu-slim permissions: - contents: - issues: + contents: read + issues: write + concurrency: + group: "gh-aw-conclusion-daily-repo-status" + cancel-in-progress: false + queue: max outputs: + incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }} noop_message: ${{ steps.noop.outputs.noop_message }} tools_reported: ${{ steps.missing_tool.outputs.tools_reported }} total_count: ${{ steps.missing_tool.outputs.total_count }} steps: - - name: Setup - uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.45. + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/ - - name: Download agent output - continue-on-error: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Repo Status" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-repo-status.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent- - path: /tmp/gh-aw/safeoutputs/ - - name: Setup agent output environment + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' run: | - mkdir -p /tmp/gh-aw/safeoutputs/ - find "/tmp/gh-aw/safeoutputs/" -type f - - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" - - name: Process No-Op - id: - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # - env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_NOOP_MAX: - GH_AW_WORKFLOW_NAME: "Daily Repo Status" - GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/d19056381ba48cb1f7c78510c23069701fa7ae87/workflows/daily-repo-status.md" + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Process no-op messages + id: noop + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_NOOP_MAX: "1" + GH_AW_WORKFLOW_NAME: "Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/blob/main/workflows/repo-status.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_NOOP_REPORT_AS_ISSUE: "true" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/noop.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_noop_message.cjs'); await main(); - - name: Record Missing - id: - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # + - name: Log detection run + id: detection_runs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_WORKFLOW_NAME: "Daily Repo Status" - GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/d19056381ba48cb1f7c78510c23069701fa7ae87/workflows/daily-repo-status.md" + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/blob/main/workflows/repo-status.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} + GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/missing_tool.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_detection_runs.cjs'); await main(); - - name: Handle Agent - id: - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # - env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_WORKFLOW_NAME: "Daily Repo Status" - GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/d19056381ba48cb1f7c78510c23069701fa7ae87/workflows/daily-repo-status.md" - GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} - GH_AW_WORKFLOW_ID: "daily-repo-status" - GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.agent.outputs.secret_verification_result }} - GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} + - name: Record missing tool + id: missing_tool + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_MISSING_TOOL_CREATE_ISSUE: "true" + GH_AW_WORKFLOW_NAME: "Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/blob/main/workflows/repo-status.md" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/handle_agent_failure.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/missing_tool.cjs'); await main(); - - name: Handle No-Op - id: - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # - env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_WORKFLOW_NAME: "Daily Repo Status" - GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/d19056381ba48cb1f7c78510c23069701fa7ae87/workflows/daily-repo-status.md" + - name: Record incomplete + id: report_incomplete + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: "true" + GH_AW_WORKFLOW_NAME: "Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/blob/main/workflows/repo-status.md" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/report_incomplete_handler.cjs'); + await main(); + - name: Handle agent failure + id: handle_agent_failure + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/blob/main/workflows/repo-status.md" GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} - GH_AW_NOOP_MESSAGE: ${{ steps.noop.outputs.noop_message }} - GH_AW_NOOP_REPORT_AS_ISSUE: "true" + GH_AW_WORKFLOW_ID: "daily-repo-status" + GH_AW_ACTION_FAILURE_ISSUE_EXPIRES_HOURS: "168" + GH_AW_ENGINE_ID: "copilot" + GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.activation.outputs.secret_verification_result }} + GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} + GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens || '' }} + GH_AW_EFFECTIVE_TOKENS_RATE_LIMIT_ERROR: ${{ needs.agent.outputs.effective_tokens_rate_limit_error || 'false' }} + GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} + GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} + GH_AW_AGENTIC_ENGINE_TIMEOUT: ${{ needs.agent.outputs.agentic_engine_timeout }} + GH_AW_MODEL_NOT_SUPPORTED_ERROR: ${{ needs.agent.outputs.model_not_supported_error }} + GH_AW_ENGINE_API_HOSTS: "api.enterprise.githubcopilot.com,api.githubcopilot.com,api.business.githubcopilot.com,api.individual.githubcopilot.com" + GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }} + GH_AW_STALE_LOCK_FILE_FAILED: ${{ needs.activation.outputs.stale_lock_file_failed }} + GH_AW_GROUP_REPORTS: "false" + GH_AW_FAILURE_REPORT_AS_ISSUE: "true" + GH_AW_MISSING_TOOL_REPORT_AS_FAILURE: "true" + GH_AW_MISSING_DATA_REPORT_AS_FAILURE: "true" + GH_AW_TIMEOUT_MINUTES: "20" + GH_AW_MAX_EFFECTIVE_TOKENS: "25000000" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/handle_noop_message.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_agent_failure.cjs'); await main(); detection: needs: - if: needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true' - runs-on: ubuntu- - permissions: {} - concurrency: - group: "gh-aw-copilot-${{ github.workflow }}" - timeout-minutes: + - activation + - agent + if: > + always() && needs.agent.result != 'skipped' && (needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true') + runs-on: ubuntu-latest + permissions: + contents: read outputs: - success: ${{ steps.parse_results.outputs.success }} + detection_conclusion: ${{ steps.detection_conclusion.outputs.conclusion }} + detection_reason: ${{ steps.detection_conclusion.outputs.reason }} + detection_success: ${{ steps.detection_conclusion.outputs.success }} steps: - - name: Setup - uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.45. + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/ - - name: Download agent - continue-on-error: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Repo Status" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-repo-status.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent- - path: /tmp/gh-aw/threat-detection/ - - name: Download agent output - continue-on-error: - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Checkout repository for patch context + if: needs.agent.outputs.has_patch == 'true' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - name: agent- - path: /tmp/gh-aw/threat-detection/ - - name: Echo agent output + persist-credentials: false + # --- Threat Detection --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3 ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca + - name: Check if detection needed + id: detection_guard + if: always() env: - AGENT_OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} + OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + run: | + if [[ -n "$OUTPUT_TYPES" || "$HAS_PATCH" == "true" ]]; then + echo "run_detection=true" >> "$GITHUB_OUTPUT" + echo "Detection will run: output_types=$OUTPUT_TYPES, has_patch=$HAS_PATCH" + else + echo "run_detection=false" >> "$GITHUB_OUTPUT" + echo "Detection skipped: no agent outputs or patches to analyze" + fi + - name: Clear MCP Config for detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' run: | - echo "Agent output-types: $AGENT_OUTPUT_TYPES" - - name: Setup threat - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # + rm -f "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json" + rm -f /home/runner/.copilot/mcp-config.json + rm -f "$GITHUB_WORKSPACE/.gemini/settings.json" + - name: Prepare threat detection files + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + mkdir -p /tmp/gh-aw/threat-detection/aw-prompts + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt 2>/dev/null || true + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/threat-detection/agent_output.json 2>/dev/null || true + for f in /tmp/gh-aw/aw-*.patch; do + [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true + done + for f in /tmp/gh-aw/aw-*.bundle; do + [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true + done + echo "Prepared threat detection files:" + ls -la /tmp/gh-aw/threat-detection/ 2>/dev/null || true + - name: Setup threat detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - WORKFLOW_NAME: "Daily Repo Status" + WORKFLOW_NAME: "Repo Status" WORKFLOW_DESCRIPTION: "This workflow creates daily repo status reports. It gathers recent repository\nactivity (issues, PRs, discussions, releases, code changes) and generates\nengaging GitHub issues with productivity insights, community highlights,\nand project recommendations." HAS_PATCH: ${{ needs.agent.outputs.has_patch }} with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/setup_threat_detection.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/setup_threat_detection.cjs'); await main(); - - name: Ensure threat-detection directory and + - name: Ensure threat-detection directory and log + if: always() && steps.detection_guard.outputs.run_detection == 'true' run: | - mkdir -p /tmp/gh-aw/threat- - touch /tmp/gh-aw/threat-detection/detection. - - name: Validate COPILOT_GITHUB_TOKEN - id: validate- - run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot- + mkdir -p /tmp/gh-aw/threat-detection + touch /tmp/gh-aw/threat-detection/detection.log + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + package-manager-cache: false + - name: Install GitHub Copilot CLI + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.52 env: - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - - name: Install GitHub Copilot - run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0. - - name: Execute GitHub Copilot - id: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.25.55 + - name: Execute GitHub Copilot CLI + if: always() && steps.detection_guard.outputs.run_detection == 'true' + continue-on-error: true + id: detection_agentic_execution # Copilot CLI tool arguments (sorted): - # --allow-tool shell(cat) - # --allow-tool shell(grep) - # --allow-tool shell(head) - # --allow-tool shell(jq) - # --allow-tool shell(ls) - # --allow-tool shell(tail) - # --allow-tool shell(wc) - timeout-minutes: + timeout-minutes: 20 run: | - set -o - COPILOT_CLI_INSTRUCTION="$(cat /tmp/gh-aw/aw-prompts/prompt.txt)" - mkdir -p /tmp/ - mkdir -p /tmp/gh-aw/ - mkdir -p /tmp/gh-aw/agent/ - mkdir -p /tmp/gh-aw/sandbox/agent/logs/ - copilot --add-dir /tmp/ --add-dir /tmp/gh-aw/ --add-dir /tmp/gh-aw/agent/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-tool 'shell(cat)' --allow-tool 'shell(grep)' --allow-tool 'shell(head)' --allow-tool 'shell(jq)' --allow-tool 'shell(ls)' --allow-tool 'shell(tail)' --allow-tool 'shell(wc)' --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$COPILOT_CLI_INSTRUCTION"${GH_AW_MODEL_DETECTION_COPILOT:+ --model "$GH_AW_MODEL_DETECTION_COPILOT"} 2>&1 | tee /tmp/gh-aw/threat-detection/detection. + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/threat-detection/detection.log) + printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.25.55/awf-config.schema.json","network":{"allowDomains":["api.business.githubcopilot.com","api.enterprise.githubcopilot.com","api.github.com","api.githubcopilot.com","api.individual.githubcopilot.com","github.com","host.docker.internal","telemetry.enterprise.githubcopilot.com"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxEffectiveTokens":25000000},"container":{"imageTag":"0.25.55"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw" + fi + # shellcheck disable=SC1003 + sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \ + -- /bin/bash -c 'export PATH="$(find /opt/hostedtoolcache /home/runner/work/_tool -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log env: - COPILOT_AGENT_RUNNER_TYPE: + AWF_REFLECT_ENABLED: 1 + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - GH_AW_MODEL_DETECTION_COPILOT: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || '' }} - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. + COPILOT_MODEL: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || 'claude-sonnet-4.6' }} + GH_AW_PHASE: detection + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_VERSION: v0.76.1 + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows GITHUB_HEAD_REF: ${{ github.head_ref }} GITHUB_REF_NAME: ${{ github.ref_name }} - GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md GITHUB_WORKSPACE: ${{ github.workspace }} - XDG_CONFIG_HOME: /home/ - - name: Parse threat detection - id: - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + XDG_CONFIG_HOME: /home/runner + - name: Upload threat detection log + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/parse_threat_detection_results.cjs'); - await main(); - - name: Upload threat detection + name: detection + path: /tmp/gh-aw/threat-detection/detection.log + if-no-files-found: ignore + - name: Parse and conclude threat detection + id: detection_conclusion if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v6.0. + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + RUN_DETECTION: ${{ steps.detection_guard.outputs.run_detection }} + DETECTION_AGENTIC_EXECUTION_OUTCOME: ${{ steps.detection_agentic_execution.outcome }} + GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" with: - name: threat-detection. - path: /tmp/gh-aw/threat-detection/detection. - if-no-files-found: + script: | + try { + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_threat_detection_results.cjs'); + await main(); + } catch (loadErr) { + const continueOnError = process.env.GH_AW_DETECTION_CONTINUE_ON_ERROR !== 'false'; + const detectionExecutionFailed = process.env.DETECTION_AGENTIC_EXECUTION_OUTCOME === 'failure'; + const msg = 'ERR_SYSTEM: \u274C Unexpected error loading threat detection module: ' + (loadErr && loadErr.message ? loadErr.message : String(loadErr)); + core.error(msg); + core.setOutput('reason', 'parse_error'); + if (continueOnError && !detectionExecutionFailed) { + core.warning('\u26A0\uFE0F ' + msg); + core.setOutput('conclusion', 'warning'); + core.setOutput('success', 'false'); + } else { + core.setOutput('conclusion', 'failure'); + core.setOutput('success', 'false'); + core.setFailed(msg); + } + } safe_outputs: needs: - - - - - if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (needs.detection.outputs.success == 'true') - runs-on: ubuntu- + - activation + - agent + - detection + if: (!cancelled()) && needs.agent.result != 'skipped' && needs.detection.result == 'success' + runs-on: ubuntu-slim permissions: - contents: - issues: - timeout-minutes: + contents: read + issues: write + timeout-minutes: 15 env: + GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/daily-repo-status" + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} + GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} + GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} GH_AW_ENGINE_ID: "copilot" + GH_AW_ENGINE_MODEL: ${{ needs.agent.outputs.model }} + GH_AW_ENGINE_VERSION: "1.0.52" GH_AW_WORKFLOW_ID: "daily-repo-status" - GH_AW_WORKFLOW_NAME: "Daily Repo Status" - GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/d19056381ba48cb1f7c78510c23069701fa7ae87/workflows/daily-repo-status.md" + GH_AW_WORKFLOW_NAME: "Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/blob/main/workflows/repo-status.md" outputs: + code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }} + code_push_failure_errors: ${{ steps.process_safe_outputs.outputs.code_push_failure_errors }} create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }} create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }} + created_issue_number: ${{ steps.process_safe_outputs.outputs.created_issue_number }} + created_issue_url: ${{ steps.process_safe_outputs.outputs.created_issue_url }} process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} steps: - - name: Setup - uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.45. + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/ - - name: Download agent output - continue-on-error: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Repo Status" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-repo-status.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent- - path: /tmp/gh-aw/safeoutputs/ - - name: Setup agent output environment + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Configure GH_HOST for enterprise compatibility + id: ghes-host-config + shell: bash run: | - mkdir -p /tmp/gh-aw/safeoutputs/ - find "/tmp/gh-aw/safeoutputs/" -type f - - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" - - name: Process Safe - id: - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # + # Derive GH_HOST from GITHUB_SERVER_URL so the gh CLI targets the correct + # GitHub instance (GHES/GHEC). On github.com this is a harmless no-op. + GH_HOST="${GITHUB_SERVER_URL#https://}" + GH_HOST="${GH_HOST#http://}" + echo "GH_HOST=${GH_HOST}" >> "$GITHUB_ENV" + - name: Process Safe Outputs + id: process_safe_outputs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_issue\":{\"labels\":[\"report\",\"daily-status\"],\"max\":1,\"title_prefix\":\"[repo-status] \"},\"missing_data\":{},\"missing_tool\":{}}" + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }} + GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_issue\":{\"close_older_issues\":true,\"labels\":[\"report\",\"daily-status\"],\"max\":1,\"title_prefix\":\"[repo-status] \"},\"create_report_incomplete_issue\":{},\"mentions\":{\"enabled\":false},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/safe_output_handler_manager.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/safe_output_handler_manager.cjs'); await main(); + - name: Upload Safe Outputs Items + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: safe-outputs-items + path: | + /tmp/gh-aw/safe-output-items.jsonl + /tmp/gh-aw/temporary-id-map.json + if-no-files-found: ignore diff --git a/.github/workflows/daily-repo-status.md b/.github/workflows/daily-repo-status.md index 431b4afb91a6..8debf5708437 100644 --- a/.github/workflows/daily-repo-status.md +++ b/.github/workflows/daily-repo-status.md @@ -22,15 +22,19 @@ tools: # reading issues, pull requests and comments from 3rd-parties # If in a private repo this has no particular effect. lockdown: false + min-integrity: none # This workflow is allowed to examine and comment on any issues safe-outputs: + mentions: false + allowed-github-references: [] create-issue: title-prefix: "[repo-status] " labels: [report, daily-status] -source: githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87 + close-older-issues: true +source: githubnext/agentics/workflows/repo-status.md@main --- -# Daily Repo Status +# Repo Status Create an upbeat daily status report for the repo as a GitHub issue. diff --git a/.github/workflows/issue-triage-agent.lock.yml b/.github/workflows/issue-triage-agent.lock.yml index 258032f02c4a..3acf91dcc1ef 100644 --- a/.github/workflows/issue-triage-agent.lock.yml +++ b/.github/workflows/issue-triage-agent.lock.yml @@ -1,4 +1,5 @@ -# +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"3cfe810a6f1f402ae902098f1dc14b99a4281a5986d58b76e96eec1a2fde7646","compiler_version":"v0.76.1","strict":true,"agent_id":"copilot"} +# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"46d564922b082d0db93244972e8005ea6904ee5f","version":"v0.76.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.55","digest":"sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55","digest":"sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.55","digest":"sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.19","digest":"sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f"},{"image":"ghcr.io/github/github-mcp-server:v1.0.4","digest":"sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4","pinned_image":"ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4"},{"image":"node:lts-alpine","digest":"sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14","pinned_image":"node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14"}]} # ___ _ _ # / _ \ | | (_) # | |_| | __ _ ___ _ __ | |_ _ ___ @@ -13,28 +14,56 @@ # \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \ # \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ # -# This file was automatically generated by gh-aw (v0.45.0). DO NOT EDIT. +# This file was automatically generated by gh-aw (v0.76.1). DO NOT EDIT. # -# To update this file, edit github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4 and run: +# To update this file, edit github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115 and run: # gh aw compile # Not all edits will cause changes to this file. # # For more information: https://github.github.com/gh-aw/introduction/overview/ # # -# Source: github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4 +# Source: github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115 # # Resolved workflow manifest: # Imports: -# - github/gh-aw/.github/workflows/shared/reporting.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4 +# - github/gh-aw/.github/workflows/shared/reporting.md@359795d49ada21681ab616bd4cbcb144a7387115 +# +# Secrets used: +# - COPILOT_GITHUB_TOKEN +# - GH_AW_GITHUB_MCP_SERVER_TOKEN +# - GH_AW_GITHUB_TOKEN +# - GITHUB_TOKEN # -# frontmatter-hash: 7bc83974fa1e47c12b40c3333872e4126711d5c6624022cc78b76047289d8b63 +# Custom actions used: +# - actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 +# - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 +# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 +# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) +# - actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 +# - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 +# - github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 +# +# Container images used: +# - ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731 +# - ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3 +# - ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca +# - ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f +# - ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4 +# - node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14 name: "Issue Triage Agent" -"on": +on: schedule: - - cron: "0 14 * * 1-5" + - cron: "49 14 * * 1-5" + # Friendly format: daily around 14:00 on weekdays (scattered) workflow_dispatch: + inputs: + aw_context: + default: "" + description: "Agent caller context (used internally by Agentic Workflows)." + required: false + type: string permissions: {} @@ -47,324 +76,503 @@ jobs: activation: runs-on: ubuntu-slim permissions: + actions: read contents: read outputs: comment_id: "" comment_repo: "" + engine_id: ${{ steps.generate_aw_info.outputs.engine_id }} + lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }} + model: ${{ steps.generate_aw_info.outputs.model }} + secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} + stale_lock_file_failed: ${{ steps.check-lock-file.outputs.stale_lock_file_failed == 'true' }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.74.8 + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/actions - - name: Check workflow file timestamps + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage-agent.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Generate agentic run info + id: generate_aw_info + env: + GH_AW_INFO_ENGINE_ID: "copilot" + GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" + GH_AW_INFO_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || 'claude-sonnet-4.6' }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AGENT_VERSION: "1.0.52" + GH_AW_INFO_CLI_VERSION: "v0.76.1" + GH_AW_INFO_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_INFO_EXPERIMENTAL: "false" + GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" + GH_AW_INFO_STAGED: "false" + GH_AW_INFO_ALLOWED_DOMAINS: '["defaults"]' + GH_AW_INFO_FIREWALL_ENABLED: "true" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_AWMG_VERSION: "" + GH_AW_INFO_FIREWALL_TYPE: "squid" + GH_AW_INFO_FRONTMATTER_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_FRONTMATTER_EMOJI: "🔧" + GH_AW_COMPILED_STRICT: "true" + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); + await main(core, context); + - name: Validate COPILOT_GITHUB_TOKEN secret + id: validate-secret + run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_multi_secret.sh" COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Checkout .github and .agents folders + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + sparse-checkout: | + .github + .agents + .antigravity + .claude + .codex + .crush + .gemini + .opencode + .pi + sparse-checkout-cone-mode: true + fetch-depth: 1 + - name: Save agent config folders for base branch restoration + env: + GH_AW_AGENT_FOLDERS: ".agents .antigravity .claude .codex .crush .gemini .github .opencode .pi" + GH_AW_AGENT_FILES: ".crush.json AGENTS.md ANTIGRAVITY.md CLAUDE.md GEMINI.md PI.md opencode.jsonc" + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/save_base_github_folders.sh" + - name: Check workflow lock file + id: check-lock-file uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_WORKFLOW_FILE: "issue-triage-agent.lock.yml" + GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}" with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/check_workflow_timestamp_api.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_timestamp_api.cjs'); + await main(); + - name: Check compile-agentic version + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_COMPILED_VERSION: "v0.76.1" + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_version_updates.cjs'); + await main(); + - name: Create prompt with built-in context + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl + GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + # poutine:ignore untrusted_checkout_exec + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" + { + cat << 'GH_AW_PROMPT_9a8f42497f411c95_EOF' + + GH_AW_PROMPT_9a8f42497f411c95_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" + cat << 'GH_AW_PROMPT_9a8f42497f411c95_EOF' + + Tools: add_comment, add_labels, missing_tool, missing_data, noop + + GH_AW_PROMPT_9a8f42497f411c95_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" + cat << 'GH_AW_PROMPT_9a8f42497f411c95_EOF' + + The following GitHub context information is available for this workflow: + {{#if github.actor}} + - **actor**: __GH_AW_GITHUB_ACTOR__ + {{/if}} + {{#if github.repository}} + - **repository**: __GH_AW_GITHUB_REPOSITORY__ + {{/if}} + {{#if github.workspace}} + - **workspace**: __GH_AW_GITHUB_WORKSPACE__ + {{/if}} + {{#if github.event.issue.number || (github.aw.context.item_type == 'issue' && github.aw.context.item_number)}} + - **issue-number**: #__GH_AW_EXPR_802A9F6A__ + {{/if}} + {{#if github.event.discussion.number || (github.aw.context.item_type == 'discussion' && github.aw.context.item_number)}} + - **discussion-number**: #__GH_AW_EXPR_1A3A194A__ + {{/if}} + {{#if github.event.pull_request.number || (github.aw.context.item_type == 'pull_request' && github.aw.context.item_number)}} + - **pull-request-number**: #__GH_AW_EXPR_463A214A__ + {{/if}} + {{#if github.event.comment.id || github.aw.context.comment_id}} + - **comment-id**: __GH_AW_EXPR_FF1D34CE__ + {{/if}} + {{#if github.run_id}} + - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ + {{/if}} + + + GH_AW_PROMPT_9a8f42497f411c95_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" + cat << 'GH_AW_PROMPT_9a8f42497f411c95_EOF' + + {{#runtime-import .github/aw/imports/github/gh-aw/359795d49ada21681ab616bd4cbcb144a7387115/.github_workflows_shared_reporting.md}} + {{#runtime-import .github/workflows/shared/noop-reminder.md}} + {{#runtime-import .github/workflows/issue-triage-agent.md}} + GH_AW_PROMPT_9a8f42497f411c95_EOF + } > "$GH_AW_PROMPT" + - name: Interpolate variables and render templates + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_ENGINE_ID: "copilot" + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/interpolate_prompt.cjs'); await main(); + - name: Substitute placeholders + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + GH_AW_MCP_CLI_SERVERS_LIST: '- `safeoutputs` — run `safeoutputs --help` to see available tools' + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + + const substitutePlaceholders = require('${{ runner.temp }}/gh-aw/actions/substitute_placeholders.cjs'); + + // Call the substitution function + return await substitutePlaceholders({ + file: process.env.GH_AW_PROMPT, + substitutions: { + GH_AW_EXPR_1A3A194A: process.env.GH_AW_EXPR_1A3A194A, + GH_AW_EXPR_463A214A: process.env.GH_AW_EXPR_463A214A, + GH_AW_EXPR_802A9F6A: process.env.GH_AW_EXPR_802A9F6A, + GH_AW_EXPR_FF1D34CE: process.env.GH_AW_EXPR_FF1D34CE, + GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, + GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, + GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, + GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, + GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST + } + }); + - name: Validate prompt placeholders + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh" + - name: Print prompt + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh" + - name: Upload activation artifact + if: success() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: activation + include-hidden-files: true + path: | + /tmp/gh-aw/aw_info.json + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/aw-prompts/prompt-template.txt + /tmp/gh-aw/aw-prompts/prompt-import-tree.json + /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/base + /tmp/gh-aw/.github/agents + /tmp/gh-aw/.github/skills + if-no-files-found: ignore + retention-days: 1 agent: needs: activation runs-on: ubuntu-latest permissions: - contents: read - issues: read + issues: read + concurrency: + group: "gh-aw-copilot-${{ github.workflow }}" env: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} GH_AW_ASSETS_ALLOWED_EXTS: "" GH_AW_ASSETS_BRANCH: "" GH_AW_ASSETS_MAX_SIZE_KB: 0 GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs - GH_AW_SAFE_OUTPUTS: /opt/gh-aw/safeoutputs/outputs.jsonl - GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json - GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json GH_AW_WORKFLOW_ID_SANITIZED: issuetriageagent outputs: + agentic_engine_timeout: ${{ steps.detect-agent-errors.outputs.agentic_engine_timeout || 'false' }} + effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }} + effective_tokens_rate_limit_error: ${{ steps.parse-mcp-gateway.outputs.effective_tokens_rate_limit_error || 'false' }} has_patch: ${{ steps.collect_output.outputs.has_patch }} - model: ${{ steps.generate_aw_info.outputs.model }} + inference_access_error: ${{ steps.detect-agent-errors.outputs.inference_access_error || 'false' }} + mcp_policy_error: ${{ steps.detect-agent-errors.outputs.mcp_policy_error || 'false' }} + model: ${{ needs.activation.outputs.model }} + model_not_supported_error: ${{ steps.detect-agent-errors.outputs.model_not_supported_error || 'false' }} output: ${{ steps.collect_output.outputs.output }} output_types: ${{ steps.collect_output.outputs.output_types }} - secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.74.8 + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/actions + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage-agent.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Set runtime paths + id: set-runtime-paths + run: | + { + echo "GH_AW_SAFE_OUTPUTS=${RUNNER_TEMP}/gh-aw/safeoutputs/outputs.jsonl" + echo "GH_AW_SAFE_OUTPUTS_CONFIG_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" + echo "GH_AW_SAFE_OUTPUTS_TOOLS_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/tools.json" + } >> "$GITHUB_OUTPUT" - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Create gh-aw temp directory - run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir.sh + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_gh_aw_tmp_dir.sh" + - name: Configure gh CLI for GitHub Enterprise + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" + env: + GH_TOKEN: ${{ github.token }} - name: Configure Git credentials env: REPO_NAME: ${{ github.repository }} SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} run: | git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" + git config --global am.keepcr true # Re-authenticate git with GitHub token SERVER_URL_STRIPPED="${SERVER_URL#https://}" - git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" echo "Git configured with standard GitHub Actions identity" - - name: Generate agentic run info - id: generate_aw_info - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - const fs = require('fs'); - - const awInfo = { - engine_id: "copilot", - engine_name: "GitHub Copilot CLI", - model: process.env.GH_AW_MODEL_AGENT_COPILOT || "", - version: "", - agent_version: "0.0.410", - cli_version: "v0.45.0", - workflow_name: "Issue Triage Agent", - experimental: false, - supports_tools_allowlist: true, - supports_http_transport: true, - run_id: context.runId, - run_number: context.runNumber, - run_attempt: process.env.GITHUB_RUN_ATTEMPT, - repository: context.repo.owner + '/' + context.repo.repo, - ref: context.ref, - sha: context.sha, - actor: context.actor, - event_name: context.eventName, - staged: false, - allowed_domains: ["defaults"], - firewall_enabled: true, - awf_version: "v0.18.0", - awmg_version: "v0.1.4", - steps: { - firewall: "squid" - }, - created_at: new Date().toISOString() - }; - - // Write to /tmp/gh-aw directory to avoid inclusion in PR - const tmpPath = '/tmp/gh-aw/aw_info.json'; - fs.writeFileSync(tmpPath, JSON.stringify(awInfo, null, 2)); - console.log('Generated aw_info.json at:', tmpPath); - console.log(JSON.stringify(awInfo, null, 2)); - - // Set model as output for reuse in other steps/jobs - core.setOutput('model', awInfo.model); - - name: Validate COPILOT_GITHUB_TOKEN secret - id: validate-secret - run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default - env: - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - name: Install GitHub Copilot CLI - run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0.410 - - name: Install awf binary - run: bash /opt/gh-aw/actions/install_awf_binary.sh v0.18.0 + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.52 + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.25.55 - name: Determine automatic lockdown mode for GitHub MCP Server id: determine-automatic-lockdown - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) env: GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} with: script: | - const determineAutomaticLockdown = require('/opt/gh-aw/actions/determine_automatic_lockdown.cjs'); + const determineAutomaticLockdown = require('${{ runner.temp }}/gh-aw/actions/determine_automatic_lockdown.cjs'); await determineAutomaticLockdown(github, context, core); + - name: Download activation artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: activation + path: /tmp/gh-aw + - name: Restore inline sub-agents from activation artifact + env: + GH_AW_SUB_AGENT_DIR: ".github/agents" + GH_AW_SUB_AGENT_EXT: ".agent.md" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_sub_agents.sh" + - name: Restore inline skills from activation artifact + env: + GH_AW_SKILL_DIR: ".github/skills" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_skills.sh" - name: Download container images - run: bash /opt/gh-aw/actions/download_docker_images.sh ghcr.io/github/gh-aw-firewall/agent:0.18.0 ghcr.io/github/gh-aw-firewall/squid:0.18.0 ghcr.io/github/gh-aw-mcpg:v0.1.4 ghcr.io/github/github-mcp-server:v0.30.3 node:lts-alpine - - name: Write Safe Outputs Config + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3 ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4 node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14 + - name: Generate Safe Outputs Config run: | - mkdir -p /opt/gh-aw/safeoutputs + mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > /opt/gh-aw/safeoutputs/config.json << 'GH_AW_SAFE_OUTPUTS_CONFIG_EOF' - {"add_comment":{"max":1},"add_labels":{"allowed":["bug","feature","enhancement","documentation","question","help-wanted","good-first-issue"],"max":3},"missing_data":{},"missing_tool":{},"noop":{"max":1}} - GH_AW_SAFE_OUTPUTS_CONFIG_EOF - cat > /opt/gh-aw/safeoutputs/tools.json << 'GH_AW_SAFE_OUTPUTS_TOOLS_EOF' - [ + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_40bb017992a44aa9_EOF' + {"add_comment":{"max":1},"add_labels":{"allowed":["bug","feature","enhancement","documentation","question","help-wanted","good-first-issue"]},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} + GH_AW_SAFE_OUTPUTS_CONFIG_40bb017992a44aa9_EOF + - name: Generate Safe Outputs Tools + env: + GH_AW_TOOLS_META_JSON: | { - "description": "Add a comment to an existing GitHub issue, pull request, or discussion. Use this to provide feedback, answer questions, or add information to an existing conversation. For creating new items, use create_issue, create_discussion, or create_pull_request instead. IMPORTANT: Comments are subject to validation constraints enforced by the MCP server - maximum 65536 characters for the complete comment (including footer which is added automatically), 10 mentions (@username), and 50 links. Exceeding these limits will result in an immediate error with specific guidance. CONSTRAINTS: Maximum 1 comment(s) can be added.", - "inputSchema": { - "additionalProperties": false, - "properties": { + "description_suffixes": { + "add_comment": " CONSTRAINTS: Maximum 1 comment(s) can be added. Supports reply_to_id for discussion threading.", + "add_labels": " CONSTRAINTS: Only these labels are allowed: [\"bug\" \"feature\" \"enhancement\" \"documentation\" \"question\" \"help-wanted\" \"good-first-issue\"]." + }, + "repo_params": {}, + "dynamic_tools": [] + } + GH_AW_VALIDATION_JSON: | + { + "add_comment": { + "defaultMax": 1, + "fields": { "body": { - "description": "The comment text in Markdown format. This is the 'body' field - do not use 'comment_body' or other variations. Provide helpful, relevant information that adds value to the conversation. CONSTRAINTS: The complete comment (your body text + automatically added footer) must not exceed 65536 characters total. Maximum 10 mentions (@username), maximum 50 links (http/https URLs). A footer (~200-500 characters) is automatically appended with workflow attribution, so leave adequate space. If these limits are exceeded, the tool call will fail with a detailed error message indicating which constraint was violated.", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 }, "item_number": { - "description": "The issue, pull request, or discussion number to comment on. This is the numeric ID from the GitHub URL (e.g., 123 in github.com/owner/repo/issues/123). If omitted, the tool will attempt to resolve the target from the current workflow context (triggering issue, PR, or discussion).", - "type": "number" + "issueOrPRNumber": true + }, + "reply_to_id": { + "type": "string", + "maxLength": 256 + }, + "repo": { + "type": "string", + "maxLength": 256 } - }, - "required": [ - "body" - ], - "type": "object" + } }, - "name": "add_comment" - }, - { - "description": "Add labels to an existing GitHub issue or pull request for categorization and filtering. Labels must already exist in the repository. For creating new issues with labels, use create_issue with the labels property instead. CONSTRAINTS: Only these labels are allowed: [bug feature enhancement documentation question help-wanted good-first-issue].", - "inputSchema": { - "additionalProperties": false, - "properties": { + "add_labels": { + "defaultMax": 5, + "fields": { "item_number": { - "description": "Issue or PR number to add labels to. This is the numeric ID from the GitHub URL (e.g., 456 in github.com/owner/repo/issues/456). If omitted, adds labels to the item that triggered this workflow.", - "type": "number" + "issueNumberOrTemporaryId": true }, "labels": { - "description": "Label names to add (e.g., ['bug', 'priority-high']). Labels must exist in the repository.", - "items": { - "type": "string" - }, - "type": "array" + "required": true, + "type": "array", + "itemType": "string", + "itemSanitize": true, + "itemMaxLength": 128 + }, + "repo": { + "type": "string", + "maxLength": 256 } - }, - "type": "object" + } }, - "name": "add_labels" - }, - { - "description": "Report that a tool or capability needed to complete the task is not available, or share any information you deem important about missing functionality or limitations. Use this when you cannot accomplish what was requested because the required functionality is missing or access is restricted.", - "inputSchema": { - "additionalProperties": false, - "properties": { + "missing_data": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "context": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "data_type": { + "type": "string", + "sanitize": true, + "maxLength": 128 + }, + "reason": { + "type": "string", + "sanitize": true, + "maxLength": 256 + } + } + }, + "missing_tool": { + "defaultMax": 20, + "fields": { "alternatives": { - "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", - "type": "string" + "type": "string", + "sanitize": true, + "maxLength": 512 }, "reason": { - "description": "Explanation of why this tool is needed or what information you want to share about the limitation (max 256 characters).", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 }, "tool": { - "description": "Optional: Name or description of the missing tool or capability (max 128 characters). Be specific about what functionality is needed.", - "type": "string" + "type": "string", + "sanitize": true, + "maxLength": 128 } - }, - "required": [ - "reason" - ], - "type": "object" + } }, - "name": "missing_tool" - }, - { - "description": "Log a transparency message when no significant actions are needed. Use this to confirm workflow completion and provide visibility when analysis is complete but no changes or outputs are required (e.g., 'No issues found', 'All checks passed'). This ensures the workflow produces human-visible output even when no other actions are taken.", - "inputSchema": { - "additionalProperties": false, - "properties": { + "noop": { + "defaultMax": 1, + "fields": { "message": { - "description": "Status or completion message to log. Should explain what was analyzed and the outcome (e.g., 'Code review complete - no issues found', 'Analysis complete - all tests passing').", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 } - }, - "required": [ - "message" - ], - "type": "object" + } }, - "name": "noop" - }, - { - "description": "Report that data or information needed to complete the task is not available. Use this when you cannot accomplish what was requested because required data, context, or information is missing.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "alternatives": { - "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", - "type": "string" - }, - "context": { - "description": "Additional context about the missing data or where it should come from (max 256 characters).", - "type": "string" - }, - "data_type": { - "description": "Type or description of the missing data or information (max 128 characters). Be specific about what data is needed.", - "type": "string" + "report_incomplete": { + "defaultMax": 5, + "fields": { + "details": { + "type": "string", + "sanitize": true, + "maxLength": 65000 }, "reason": { - "description": "Explanation of why this data is needed to complete the task (max 256 characters).", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 1024 } - }, - "required": [], - "type": "object" - }, - "name": "missing_data" - } - ] - GH_AW_SAFE_OUTPUTS_TOOLS_EOF - cat > /opt/gh-aw/safeoutputs/validation.json << 'GH_AW_SAFE_OUTPUTS_VALIDATION_EOF' - { - "add_comment": { - "defaultMax": 1, - "fields": { - "body": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 65000 - }, - "item_number": { - "issueOrPRNumber": true - } - } - }, - "add_labels": { - "defaultMax": 5, - "fields": { - "item_number": { - "issueOrPRNumber": true - }, - "labels": { - "required": true, - "type": "array", - "itemType": "string", - "itemSanitize": true, - "itemMaxLength": 128 - } - } - }, - "missing_tool": { - "defaultMax": 20, - "fields": { - "alternatives": { - "type": "string", - "sanitize": true, - "maxLength": 512 - }, - "reason": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 256 - }, - "tool": { - "type": "string", - "sanitize": true, - "maxLength": 128 - } - } - }, - "noop": { - "defaultMax": 1, - "fields": { - "message": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 65000 } } } - } - GH_AW_SAFE_OUTPUTS_VALIDATION_EOF + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_safe_outputs_tools.cjs'); + await main(); - name: Generate Safe Outputs MCP Server Config id: safe-outputs-config run: | @@ -387,59 +595,81 @@ jobs: id: safe-outputs-start env: DEBUG: '*' + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-config.outputs.safe_outputs_port }} GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-config.outputs.safe_outputs_api_key }} - GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json - GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: ${{ runner.temp }}/gh-aw/safeoutputs/tools.json + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: ${{ runner.temp }}/gh-aw/safeoutputs/config.json GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs run: | # Environment variables are set above to prevent template injection export DEBUG + export GH_AW_SAFE_OUTPUTS export GH_AW_SAFE_OUTPUTS_PORT export GH_AW_SAFE_OUTPUTS_API_KEY export GH_AW_SAFE_OUTPUTS_TOOLS_PATH export GH_AW_SAFE_OUTPUTS_CONFIG_PATH export GH_AW_MCP_LOG_DIR - bash /opt/gh-aw/actions/start_safe_outputs_server.sh + bash "${RUNNER_TEMP}/gh-aw/actions/start_safe_outputs_server.sh" - name: Start MCP Gateway id: start-mcp-gateway env: - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-start.outputs.api_key }} GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-start.outputs.port }} - GITHUB_MCP_LOCKDOWN: ${{ steps.determine-automatic-lockdown.outputs.lockdown == 'true' && '1' || '0' }} + GITHUB_MCP_GUARD_MIN_INTEGRITY: ${{ steps.determine-automatic-lockdown.outputs.min_integrity }} + GITHUB_MCP_GUARD_REPOS: ${{ steps.determine-automatic-lockdown.outputs.repos }} GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} run: | set -eo pipefail - mkdir -p /tmp/gh-aw/mcp-config + mkdir -p "${RUNNER_TEMP}/gh-aw/mcp-config" # Export gateway environment variables for MCP config and gateway script - export MCP_GATEWAY_PORT="80" + export MCP_GATEWAY_PORT="8080" export MCP_GATEWAY_DOMAIN="host.docker.internal" + export MCP_GATEWAY_HOST_DOMAIN="localhost" MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') echo "::add-mask::${MCP_GATEWAY_API_KEY}" export MCP_GATEWAY_API_KEY export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" + export MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD="524288" export DEBUG="*" export GH_AW_ENGINE="copilot" - export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_LOCKDOWN -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.1.4' + export GH_AW_MCP_CLI_SERVERS='["safeoutputs"]' + echo GH_AW_MCP_CLI_SERVERS='["safeoutputs"]' >> "$GITHUB_ENV" + MCP_GATEWAY_UID=$(id -u 2>/dev/null || echo '0') + MCP_GATEWAY_GID=$(id -g 2>/dev/null || echo '0') + case "${DOCKER_HOST:-}" in + unix://* ) DOCKER_SOCK_PATH="${DOCKER_HOST#unix://}" ;; + /* ) DOCKER_SOCK_PATH="$DOCKER_HOST" ;; + * ) DOCKER_SOCK_PATH=/var/run/docker.sock ;; + esac + DOCKER_SOCK_GID=$(stat -c '%g' "$DOCKER_SOCK_PATH" 2>/dev/null || echo '0') + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host --add-host host.docker.internal:127.0.0.1 --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.3.19' mkdir -p /home/runner/.copilot - cat << GH_AW_MCP_CONFIG_EOF | bash /opt/gh-aw/actions/start_mcp_gateway.sh + GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) + cat << GH_AW_MCP_CONFIG_8874a73d1c6a94b7_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "github": { "type": "stdio", - "container": "ghcr.io/github/github-mcp-server:v0.30.3", + "container": "ghcr.io/github/github-mcp-server:v1.0.4", "env": { - "GITHUB_LOCKDOWN_MODE": "$GITHUB_MCP_LOCKDOWN", + "GITHUB_HOST": "\${GITHUB_SERVER_URL}", "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", "GITHUB_READ_ONLY": "1", "GITHUB_TOOLSETS": "issues,labels" + }, + "guard-policies": { + "allow-only": { + "min-integrity": "$GITHUB_MCP_GUARD_MIN_INTEGRITY", + "repos": "$GITHUB_MCP_GUARD_REPOS" + } } }, "safeoutputs": { @@ -447,6 +677,13 @@ jobs: "url": "http://host.docker.internal:$GH_AW_SAFE_OUTPUTS_PORT", "headers": { "Authorization": "\${GH_AW_SAFE_OUTPUTS_API_KEY}" + }, + "guard-policies": { + "write-sink": { + "accept": [ + "*" + ] + } } } }, @@ -457,199 +694,96 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_EOF - - name: Generate workflow overview - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - const { generateWorkflowOverview } = require('/opt/gh-aw/actions/generate_workflow_overview.cjs'); - await generateWorkflowOverview(core); - - name: Create prompt with built-in context - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} - GH_AW_GITHUB_ACTOR: ${{ github.actor }} - GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - run: | - bash /opt/gh-aw/actions/create_prompt_first.sh - cat << 'GH_AW_PROMPT_EOF' > "$GH_AW_PROMPT" - - GH_AW_PROMPT_EOF - cat "/opt/gh-aw/prompts/xpia.md" >> "$GH_AW_PROMPT" - cat "/opt/gh-aw/prompts/temp_folder_prompt.md" >> "$GH_AW_PROMPT" - cat "/opt/gh-aw/prompts/markdown.md" >> "$GH_AW_PROMPT" - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - - GitHub API Access Instructions - - The gh CLI is NOT authenticated. Do NOT use gh commands for GitHub operations. - - - To create or modify GitHub resources (issues, discussions, pull requests, etc.), you MUST call the appropriate safe output tool. Simply writing content will NOT work - the workflow requires actual tool calls. - - Temporary IDs: Some safe output tools support a temporary ID field (usually named temporary_id) so you can reference newly-created items elsewhere in the SAME agent output (for example, using #aw_abc1 in a later body). - - **IMPORTANT - temporary_id format rules:** - - If you DON'T need to reference the item later, OMIT the temporary_id field entirely (it will be auto-generated if needed) - - If you DO need cross-references/chaining, you MUST match this EXACT validation regex: /^aw_[A-Za-z0-9]{3,8}$/i - - Format: aw_ prefix followed by 3 to 8 alphanumeric characters (A-Z, a-z, 0-9, case-insensitive) - - Valid alphanumeric characters: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 - - INVALID examples: aw_ab (too short), aw_123456789 (too long), aw_test-id (contains hyphen), aw_id_123 (contains underscore) - - VALID examples: aw_abc, aw_abc1, aw_Test123, aw_A1B2C3D4, aw_12345678 - - To generate valid IDs: use 3-8 random alphanumeric characters or omit the field to let the system auto-generate - - Do NOT invent other aw_* formats — downstream steps will reject them with validation errors matching against /^aw_[A-Za-z0-9]{3,8}$/i. - - Discover available tools from the safeoutputs MCP server. - - **Critical**: Tool calls write structured data that downstream jobs process. Without tool calls, follow-up actions will be skipped. - - **Note**: If you made no other safe output tool calls during this workflow execution, call the "noop" tool to provide a status message indicating completion or that no actions were needed. - - - - The following GitHub context information is available for this workflow: - {{#if __GH_AW_GITHUB_ACTOR__ }} - - **actor**: __GH_AW_GITHUB_ACTOR__ - {{/if}} - {{#if __GH_AW_GITHUB_REPOSITORY__ }} - - **repository**: __GH_AW_GITHUB_REPOSITORY__ - {{/if}} - {{#if __GH_AW_GITHUB_WORKSPACE__ }} - - **workspace**: __GH_AW_GITHUB_WORKSPACE__ - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ }} - - **issue-number**: #__GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ }} - - **discussion-number**: #__GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ }} - - **pull-request-number**: #__GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_COMMENT_ID__ }} - - **comment-id**: __GH_AW_GITHUB_EVENT_COMMENT_ID__ - {{/if}} - {{#if __GH_AW_GITHUB_RUN_ID__ }} - - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ - {{/if}} - - - GH_AW_PROMPT_EOF - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - - GH_AW_PROMPT_EOF - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - {{#runtime-import .github/aw/imports/github/gh-aw/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github_workflows_shared_reporting.md}} - GH_AW_PROMPT_EOF - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - {{#runtime-import .github/workflows/issue-triage-agent.md}} - GH_AW_PROMPT_EOF - - name: Substitute placeholders - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + GH_AW_MCP_CONFIG_8874a73d1c6a94b7_EOF + - name: Mount MCP servers as CLIs + id: mount-mcp-clis + continue-on-error: true env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_GITHUB_ACTOR: ${{ github.actor }} - GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - with: - script: | - const substitutePlaceholders = require('/opt/gh-aw/actions/substitute_placeholders.cjs'); - - // Call the substitution function - return await substitutePlaceholders({ - file: process.env.GH_AW_PROMPT, - substitutions: { - GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, - GH_AW_GITHUB_EVENT_COMMENT_ID: process.env.GH_AW_GITHUB_EVENT_COMMENT_ID, - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: process.env.GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER, - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: process.env.GH_AW_GITHUB_EVENT_ISSUE_NUMBER, - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: process.env.GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER, - GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, - GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, - GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE - } - }); - - name: Interpolate variables and render templates + MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + MCP_GATEWAY_DOMAIN: ${{ steps.start-mcp-gateway.outputs.gateway-domain }} + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/interpolate_prompt.cjs'); + const { main } = require('${{ runner.temp }}/gh-aw/actions/mount_mcp_as_cli.cjs'); await main(); - - name: Validate prompt placeholders - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - run: bash /opt/gh-aw/actions/validate_prompt_placeholders.sh - - name: Print prompt - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - run: bash /opt/gh-aw/actions/print_prompt_summary.sh - - name: Clean git credentials - run: bash /opt/gh-aw/actions/clean_git_credentials.sh + - name: Clean credentials + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/clean_git_credentials.sh" + - name: Audit pre-agent workspace + id: pre_agent_audit + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/audit_pre_agent_workspace.sh" - name: Execute GitHub Copilot CLI id: agentic_execution # Copilot CLI tool arguments (sorted): timeout-minutes: 5 run: | set -o pipefail - sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.18.0 --skip-pull \ - -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --add-dir "${GITHUB_WORKSPACE}" --disable-builtin-mcps --allow-all-tools --allow-all-paths --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"${GH_AW_MODEL_AGENT_COPILOT:+ --model "$GH_AW_MODEL_AGENT_COPILOT"}' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/agent-stdio.log) + printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.25.55/awf-config.schema.json","network":{"allowDomains":["api.business.githubcopilot.com","api.enterprise.githubcopilot.com","api.github.com","api.githubcopilot.com","api.individual.githubcopilot.com","api.snapcraft.io","archive.ubuntu.com","azure.archive.ubuntu.com","crl.geotrust.com","crl.globalsign.com","crl.identrust.com","crl.sectigo.com","crl.thawte.com","crl.usertrust.com","crl.verisign.com","crl3.digicert.com","crl4.digicert.com","crls.ssl.com","github.com","host.docker.internal","json-schema.org","json.schemastore.org","keyserver.ubuntu.com","ocsp.digicert.com","ocsp.geotrust.com","ocsp.globalsign.com","ocsp.identrust.com","ocsp.sectigo.com","ocsp.ssl.com","ocsp.thawte.com","ocsp.usertrust.com","ocsp.verisign.com","packagecloud.io","packages.cloud.google.com","packages.microsoft.com","ppa.launchpad.net","raw.githubusercontent.com","registry.npmjs.org","s.symcb.com","s.symcd.com","security.ubuntu.com","telemetry.enterprise.githubcopilot.com","ts-crl.ws.symantec.com","ts-ocsp.ws.symantec.com","www.googleapis.com"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxEffectiveTokens":25000000,"models":{"agent":["sonnet-6x","gpt-5.4","gpt-5.3","gemini-pro","any"],"antigravity":["copilot/antigravity*","google/antigravity*","gemini/antigravity*"],"any":["copilot/*","anthropic/*","openai/*","google/*","gemini/*"],"claude":["agent"],"codex":["agent"],"coding":["copilot/gpt-5*codex*","openai/gpt-5*codex*","gpt-5-codex"],"computer-use":["copilot/*computer-use*","google/*computer-use*","gemini/*computer-use*","openai/*computer-use*"],"copilot":["agent"],"deep-research":["copilot/deep-research*","copilot/o3-deep-research*","copilot/o4-mini-deep-research*","google/deep-research*","gemini/deep-research*","openai/o3-deep-research*","openai/o4-mini-deep-research*"],"gemini":["agent"],"gemini-3-flash":["copilot/gemini-3*flash*","google/gemini-3*flash*","gemini/gemini-3*flash*"],"gemini-3-pro":["copilot/gemini-3*pro*","google/gemini-3*pro*","gemini/gemini-3*pro*"],"gemini-3.1-flash":["copilot/gemini-3.1*flash*","google/gemini-3.1*flash*","gemini/gemini-3.1*flash*"],"gemini-3.1-pro":["copilot/gemini-3.1*pro*","google/gemini-3.1*pro*","gemini/gemini-3.1*pro*"],"gemini-3.5-flash":["copilot/gemini-3.5*flash*","google/gemini-3.5*flash*","gemini/gemini-3.5*flash*"],"gemini-flash":["copilot/gemini-*flash*","google/gemini-*flash*","gemini/gemini-*flash*"],"gemini-flash-lite":["copilot/gemini-*flash*lite*","google/gemini-*flash*lite*","gemini/gemini-*flash*lite*"],"gemini-pro":["copilot/gemini-*pro*","google/gemini-*pro*","gemini/gemini-*pro*"],"gemma":["copilot/gemma*","google/gemma*","gemini/gemma*"],"gpt-4.1":["copilot/gpt-4.1*","openai/gpt-4.1*"],"gpt-5":["copilot/gpt-5*","openai/gpt-5*"],"gpt-5-codex":["copilot/gpt-5*codex*","openai/gpt-5*codex*"],"gpt-5-mini":["copilot/gpt-5*mini*","openai/gpt-5*mini*"],"gpt-5-nano":["copilot/gpt-5*nano*","openai/gpt-5*nano*"],"gpt-5-pro":["copilot/gpt-5*pro*","openai/gpt-5*pro*"],"gpt-5.2":["copilot/gpt-5.2*","openai/gpt-5.2*"],"gpt-5.3":["copilot/gpt-5.3*","openai/gpt-5.3*"],"gpt-5.4":["copilot/gpt-5.4*","openai/gpt-5.4*"],"gpt-5.5":["copilot/gpt-5.5*","openai/gpt-5.5*"],"haiku":["copilot/*haiku*","anthropic/*haiku*"],"large":["sonnet","gpt-5-pro","gpt-5","gemini-pro"],"mini":["haiku","gpt-5-mini","gpt-5-nano","gemini-flash-lite"],"opus":["copilot/*opus*","anthropic/*opus*"],"opusplan":["opus?effort=high"],"reasoning":["copilot/o1*","copilot/o3*","copilot/o4*","openai/o1*","openai/o3*","openai/o4*"],"robotics":["copilot/*robotics*","google/*robotics*","gemini/*robotics*"],"small":["mini"],"sonnet":["copilot/*sonnet*","anthropic/*sonnet*"],"sonnet-6x":["copilot/*sonnet-4-5-*","anthropic/*sonnet-4-5-*","copilot/*sonnet-4-6*","anthropic/*sonnet-4-6*"],"summarization":["haiku","gpt-5-mini","gemini-flash-lite","mini"],"vision":["copilot/gemini-*image*","gemini/gemini-*image*","copilot/gemini-*flash*","gemini/gemini-*flash*"]}},"container":{"imageTag":"0.25.55,squid=sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca,agent=sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731,api-proxy=sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw" + fi + # shellcheck disable=SC1003 + sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \ + -- /bin/bash -c 'export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && export PATH="$(find /opt/hostedtoolcache /home/runner/work/_tool -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: + AWF_REFLECT_ENABLED: 1 COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || 'claude-sonnet-4.6' }} GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json - GH_AW_MODEL_AGENT_COPILOT: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }} + GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_VERSION: v0.76.1 + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} GITHUB_REF_NAME: ${{ github.ref_name }} - GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] XDG_CONFIG_HOME: /home/runner + - name: Detect agent errors + if: always() + id: detect-agent-errors + continue-on-error: true + run: node "${RUNNER_TEMP}/gh-aw/actions/detect_agent_errors.cjs" - name: Configure Git credentials env: REPO_NAME: ${{ github.repository }} SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} run: | git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" + git config --global am.keepcr true # Re-authenticate git with GitHub token SERVER_URL_STRIPPED="${SERVER_URL#https://}" - git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" echo "Git configured with standard GitHub Actions identity" - name: Copy Copilot session state files to logs if: always() continue-on-error: true - run: | - # Copy Copilot session state files to logs folder for artifact collection - # This ensures they are in /tmp/gh-aw/ where secret redaction can scan them - SESSION_STATE_DIR="$HOME/.copilot/session-state" - LOGS_DIR="/tmp/gh-aw/sandbox/agent/logs" - - if [ -d "$SESSION_STATE_DIR" ]; then - echo "Copying Copilot session state files from $SESSION_STATE_DIR to $LOGS_DIR" - mkdir -p "$LOGS_DIR" - cp -v "$SESSION_STATE_DIR"/*.jsonl "$LOGS_DIR/" 2>/dev/null || true - echo "Session state files copied successfully" - else - echo "No session-state directory found at $SESSION_STATE_DIR" - fi + run: bash "${RUNNER_TEMP}/gh-aw/actions/copy_copilot_session_state.sh" - name: Stop MCP Gateway if: always() continue-on-error: true @@ -658,15 +792,15 @@ jobs: MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} run: | - bash /opt/gh-aw/actions/stop_mcp_gateway.sh "$GATEWAY_PID" + bash "${RUNNER_TEMP}/gh-aw/actions/stop_mcp_gateway.sh" "$GATEWAY_PID" - name: Redact secrets in logs if: always() uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/redact_secrets.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_secrets.cjs'); await main(); env: GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' @@ -674,43 +808,31 @@ jobs: SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Upload Safe Outputs + - name: Append agent step summary if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: safe-output - path: ${{ env.GH_AW_SAFE_OUTPUTS }} - if-no-files-found: warn + run: bash "${RUNNER_TEMP}/gh-aw/actions/append_agent_step_summary.sh" + - name: Copy Safe Outputs + if: always() + env: + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + run: | + mkdir -p /tmp/gh-aw + cp "$GH_AW_SAFE_OUTPUTS" /tmp/gh-aw/safeoutputs.jsonl 2>/dev/null || true - name: Ingest agent output id: collect_output if: always() uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} - GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/collect_ndjson_output.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/collect_ndjson_output.cjs'); await main(); - - name: Upload sanitized agent output - if: always() && env.GH_AW_AGENT_OUTPUT - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: agent-output - path: ${{ env.GH_AW_AGENT_OUTPUT }} - if-no-files-found: warn - - name: Upload engine output files - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: agent_outputs - path: | - /tmp/gh-aw/sandbox/agent/logs/ - /tmp/gh-aw/redacted-urls.log - if-no-files-found: ignore - name: Parse agent logs for step summary if: always() uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 @@ -718,18 +840,19 @@ jobs: GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/parse_copilot_log.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_copilot_log.cjs'); await main(); - name: Parse MCP Gateway logs for step summary if: always() + id: parse-mcp-gateway uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/parse_mcp_gateway_log.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); await main(); - name: Print firewall logs if: always() @@ -737,28 +860,65 @@ jobs: env: AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs run: | - # Fix permissions on firewall logs so they can be uploaded as artifacts + # Fix permissions on firewall logs/audit dirs so they can be uploaded as artifacts # AWF runs with sudo, creating files owned by root - sudo chmod -R a+r /tmp/gh-aw/sandbox/firewall/logs 2>/dev/null || true + sudo chmod -R a+rX /tmp/gh-aw/sandbox/firewall 2>/dev/null || true # Only run awf logs summary if awf command exists (it may not be installed if workflow failed before install step) if command -v awf &> /dev/null; then awf logs summary | tee -a "$GITHUB_STEP_SUMMARY" else echo 'AWF binary not installed, skipping firewall log summary' fi + - name: Parse token usage for step summary + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_token_usage.cjs'); + await main(); + - name: Print AWF reflect summary + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/awf_reflect_summary.cjs'); + await main(); + - name: Write agent output placeholder if missing + if: always() + run: | + if [ ! -f /tmp/gh-aw/agent_output.json ]; then + echo '{"items":[]}' > /tmp/gh-aw/agent_output.json + fi - name: Upload agent artifacts if: always() continue-on-error: true uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: agent-artifacts + name: agent path: | /tmp/gh-aw/aw-prompts/prompt.txt - /tmp/gh-aw/aw_info.json + /tmp/gh-aw/sandbox/agent/logs/ + /tmp/gh-aw/redacted-urls.log /tmp/gh-aw/mcp-logs/ - /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/agent_usage.json /tmp/gh-aw/agent-stdio.log + /tmp/gh-aw/pre-agent-audit.txt /tmp/gh-aw/agent/ + /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/safeoutputs.jsonl + /tmp/gh-aw/agent_output.json + /tmp/gh-aw/aw-*.patch + /tmp/gh-aw/aw-*.bundle + /tmp/gh-aw/awf-config.json + /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/sandbox/firewall/audit/ + /tmp/gh-aw/sandbox/firewall/awf-reflect.json if-no-files-found: ignore conclusion: @@ -767,134 +927,252 @@ jobs: - agent - detection - safe_outputs - if: (always()) && (needs.agent.result != 'skipped') + if: > + always() && (needs.agent.result != 'skipped' || needs.activation.outputs.lockdown_check_failed == 'true' || + needs.activation.outputs.stale_lock_file_failed == 'true') runs-on: ubuntu-slim permissions: contents: read discussions: write issues: write pull-requests: write + concurrency: + group: "gh-aw-conclusion-issue-triage-agent" + cancel-in-progress: false + queue: max outputs: + incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }} noop_message: ${{ steps.noop.outputs.noop_message }} tools_reported: ${{ steps.missing_tool.outputs.tools_reported }} total_count: ${{ steps.missing_tool.outputs.total_count }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.74.8 + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/actions + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage-agent.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" - name: Download agent output artifact + id: download-agent-output continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent-output - path: /tmp/gh-aw/safeoutputs/ + name: agent + path: /tmp/gh-aw/ - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' run: | - mkdir -p /tmp/gh-aw/safeoutputs/ - find "/tmp/gh-aw/safeoutputs/" -type f -print - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" - - name: Process No-Op Messages + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Process no-op messages id: noop uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_NOOP_MAX: 1 + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_NOOP_MAX: "1" GH_AW_WORKFLOW_NAME: "Issue Triage Agent" - GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/tree/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github/workflows/issue-triage-agent.md" + GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/blob/359795d49ada21681ab616bd4cbcb144a7387115/.github/workflows/issue-triage-agent.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_NOOP_REPORT_AS_ISSUE: "true" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/noop.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_noop_message.cjs'); + await main(); + - name: Log detection run + id: detection_runs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/blob/359795d49ada21681ab616bd4cbcb144a7387115/.github/workflows/issue-triage-agent.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} + GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_detection_runs.cjs'); await main(); - - name: Record Missing Tool + - name: Record missing tool id: missing_tool uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_MISSING_TOOL_CREATE_ISSUE: "true" GH_AW_WORKFLOW_NAME: "Issue Triage Agent" - GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/tree/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github/workflows/issue-triage-agent.md" + GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/blob/359795d49ada21681ab616bd4cbcb144a7387115/.github/workflows/issue-triage-agent.md" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/missing_tool.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/missing_tool.cjs'); await main(); - - name: Handle Agent Failure - id: handle_agent_failure + - name: Record incomplete + id: report_incomplete uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: "true" GH_AW_WORKFLOW_NAME: "Issue Triage Agent" - GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/tree/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github/workflows/issue-triage-agent.md" - GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} - GH_AW_WORKFLOW_ID: "issue-triage-agent" - GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.agent.outputs.secret_verification_result }} + GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/blob/359795d49ada21681ab616bd4cbcb144a7387115/.github/workflows/issue-triage-agent.md" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/handle_agent_failure.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/report_incomplete_handler.cjs'); await main(); - - name: Handle No-Op Message - id: handle_noop_message + - name: Handle agent failure + id: handle_agent_failure + if: always() uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} GH_AW_WORKFLOW_NAME: "Issue Triage Agent" - GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/tree/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github/workflows/issue-triage-agent.md" + GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/blob/359795d49ada21681ab616bd4cbcb144a7387115/.github/workflows/issue-triage-agent.md" GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} - GH_AW_NOOP_MESSAGE: ${{ steps.noop.outputs.noop_message }} - GH_AW_NOOP_REPORT_AS_ISSUE: "true" + GH_AW_WORKFLOW_ID: "issue-triage-agent" + GH_AW_ACTION_FAILURE_ISSUE_EXPIRES_HOURS: "168" + GH_AW_ENGINE_ID: "copilot" + GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.activation.outputs.secret_verification_result }} + GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens || '' }} + GH_AW_EFFECTIVE_TOKENS_RATE_LIMIT_ERROR: ${{ needs.agent.outputs.effective_tokens_rate_limit_error || 'false' }} + GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} + GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} + GH_AW_AGENTIC_ENGINE_TIMEOUT: ${{ needs.agent.outputs.agentic_engine_timeout }} + GH_AW_MODEL_NOT_SUPPORTED_ERROR: ${{ needs.agent.outputs.model_not_supported_error }} + GH_AW_ENGINE_API_HOSTS: "api.enterprise.githubcopilot.com,api.githubcopilot.com,api.business.githubcopilot.com,api.individual.githubcopilot.com" + GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }} + GH_AW_STALE_LOCK_FILE_FAILED: ${{ needs.activation.outputs.stale_lock_file_failed }} + GH_AW_GROUP_REPORTS: "false" + GH_AW_FAILURE_REPORT_AS_ISSUE: "true" + GH_AW_MISSING_TOOL_REPORT_AS_FAILURE: "true" + GH_AW_MISSING_DATA_REPORT_AS_FAILURE: "true" + GH_AW_TIMEOUT_MINUTES: "5" + GH_AW_MAX_EFFECTIVE_TOKENS: "25000000" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/handle_noop_message.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_agent_failure.cjs'); await main(); detection: - needs: agent - if: needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true' + needs: + - activation + - agent + if: > + always() && needs.agent.result != 'skipped' && (needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true') runs-on: ubuntu-latest - permissions: {} - timeout-minutes: 10 + permissions: + contents: read outputs: - success: ${{ steps.parse_results.outputs.success }} + detection_conclusion: ${{ steps.detection_conclusion.outputs.conclusion }} + detection_reason: ${{ steps.detection_conclusion.outputs.reason }} + detection_success: ${{ steps.detection_conclusion.outputs.success }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.74.8 - with: - destination: /opt/gh-aw/actions - - name: Download agent artifacts - continue-on-error: true - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - name: agent-artifacts - path: /tmp/gh-aw/threat-detection/ + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage-agent.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" - name: Download agent output artifact + id: download-agent-output continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent-output - path: /tmp/gh-aw/threat-detection/ - - name: Echo agent output types + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Checkout repository for patch context + if: needs.agent.outputs.has_patch == 'true' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + # --- Threat Detection --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3 ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca + - name: Check if detection needed + id: detection_guard + if: always() env: - AGENT_OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} + OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + run: | + if [[ -n "$OUTPUT_TYPES" || "$HAS_PATCH" == "true" ]]; then + echo "run_detection=true" >> "$GITHUB_OUTPUT" + echo "Detection will run: output_types=$OUTPUT_TYPES, has_patch=$HAS_PATCH" + else + echo "run_detection=false" >> "$GITHUB_OUTPUT" + echo "Detection skipped: no agent outputs or patches to analyze" + fi + - name: Clear MCP Config for detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + rm -f "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json" + rm -f /home/runner/.copilot/mcp-config.json + rm -f "$GITHUB_WORKSPACE/.gemini/settings.json" + - name: Prepare threat detection files + if: always() && steps.detection_guard.outputs.run_detection == 'true' run: | - echo "Agent output-types: $AGENT_OUTPUT_TYPES" + mkdir -p /tmp/gh-aw/threat-detection/aw-prompts + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt 2>/dev/null || true + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/threat-detection/agent_output.json 2>/dev/null || true + for f in /tmp/gh-aw/aw-*.patch; do + [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true + done + for f in /tmp/gh-aw/aw-*.bundle; do + [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true + done + echo "Prepared threat detection files:" + ls -la /tmp/gh-aw/threat-detection/ 2>/dev/null || true - name: Setup threat detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: WORKFLOW_NAME: "Issue Triage Agent" @@ -902,72 +1180,117 @@ jobs: HAS_PATCH: ${{ needs.agent.outputs.has_patch }} with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/setup_threat_detection.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/setup_threat_detection.cjs'); await main(); - name: Ensure threat-detection directory and log + if: always() && steps.detection_guard.outputs.run_detection == 'true' run: | mkdir -p /tmp/gh-aw/threat-detection touch /tmp/gh-aw/threat-detection/detection.log - - name: Validate COPILOT_GITHUB_TOKEN secret - id: validate-secret - run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default - env: - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + package-manager-cache: false - name: Install GitHub Copilot CLI - run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0.410 + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.52 + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.25.55 - name: Execute GitHub Copilot CLI - id: agentic_execution + if: always() && steps.detection_guard.outputs.run_detection == 'true' + continue-on-error: true + id: detection_agentic_execution # Copilot CLI tool arguments (sorted): - # --allow-tool shell(cat) - # --allow-tool shell(grep) - # --allow-tool shell(head) - # --allow-tool shell(jq) - # --allow-tool shell(ls) - # --allow-tool shell(tail) - # --allow-tool shell(wc) timeout-minutes: 20 run: | set -o pipefail - COPILOT_CLI_INSTRUCTION="$(cat /tmp/gh-aw/aw-prompts/prompt.txt)" - mkdir -p /tmp/ - mkdir -p /tmp/gh-aw/ - mkdir -p /tmp/gh-aw/agent/ - mkdir -p /tmp/gh-aw/sandbox/agent/logs/ - copilot --add-dir /tmp/ --add-dir /tmp/gh-aw/ --add-dir /tmp/gh-aw/agent/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-tool 'shell(cat)' --allow-tool 'shell(grep)' --allow-tool 'shell(head)' --allow-tool 'shell(jq)' --allow-tool 'shell(ls)' --allow-tool 'shell(tail)' --allow-tool 'shell(wc)' --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$COPILOT_CLI_INSTRUCTION"${GH_AW_MODEL_DETECTION_COPILOT:+ --model "$GH_AW_MODEL_DETECTION_COPILOT"} 2>&1 | tee /tmp/gh-aw/threat-detection/detection.log + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/threat-detection/detection.log) + printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.25.55/awf-config.schema.json","network":{"allowDomains":["api.business.githubcopilot.com","api.enterprise.githubcopilot.com","api.github.com","api.githubcopilot.com","api.individual.githubcopilot.com","github.com","host.docker.internal","telemetry.enterprise.githubcopilot.com"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxEffectiveTokens":25000000},"container":{"imageTag":"0.25.55"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw" + fi + # shellcheck disable=SC1003 + sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \ + -- /bin/bash -c 'export PATH="$(find /opt/hostedtoolcache /home/runner/work/_tool -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log env: + AWF_REFLECT_ENABLED: 1 COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - GH_AW_MODEL_DETECTION_COPILOT: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || '' }} + COPILOT_MODEL: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || 'claude-sonnet-4.6' }} + GH_AW_PHASE: detection GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_VERSION: v0.76.1 + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows GITHUB_HEAD_REF: ${{ github.head_ref }} GITHUB_REF_NAME: ${{ github.ref_name }} - GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] XDG_CONFIG_HOME: /home/runner - - name: Parse threat detection results - id: parse_results - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/parse_threat_detection_results.cjs'); - await main(); - name: Upload threat detection log - if: always() + if: always() && steps.detection_guard.outputs.run_detection == 'true' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: threat-detection.log + name: detection path: /tmp/gh-aw/threat-detection/detection.log if-no-files-found: ignore + - name: Parse and conclude threat detection + id: detection_conclusion + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + RUN_DETECTION: ${{ steps.detection_guard.outputs.run_detection }} + DETECTION_AGENTIC_EXECUTION_OUTCOME: ${{ steps.detection_agentic_execution.outcome }} + GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" + with: + script: | + try { + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_threat_detection_results.cjs'); + await main(); + } catch (loadErr) { + const continueOnError = process.env.GH_AW_DETECTION_CONTINUE_ON_ERROR !== 'false'; + const detectionExecutionFailed = process.env.DETECTION_AGENTIC_EXECUTION_OUTCOME === 'failure'; + const msg = 'ERR_SYSTEM: \u274C Unexpected error loading threat detection module: ' + (loadErr && loadErr.message ? loadErr.message : String(loadErr)); + core.error(msg); + core.setOutput('reason', 'parse_error'); + if (continueOnError && !detectionExecutionFailed) { + core.warning('\u26A0\uFE0F ' + msg); + core.setOutput('conclusion', 'warning'); + core.setOutput('success', 'false'); + } else { + core.setOutput('conclusion', 'failure'); + core.setOutput('success', 'false'); + core.setFailed(msg); + } + } safe_outputs: needs: + - activation - agent - detection - if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (needs.detection.outputs.success == 'true') + if: (!cancelled()) && needs.agent.result != 'skipped' && needs.detection.result == 'success' runs-on: ubuntu-slim permissions: contents: read @@ -976,41 +1299,89 @@ jobs: pull-requests: write timeout-minutes: 15 env: + GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/issue-triage-agent" + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} + GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} + GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} + GH_AW_ENGINE_ID: "copilot" + GH_AW_ENGINE_MODEL: ${{ needs.agent.outputs.model }} + GH_AW_ENGINE_VERSION: "1.0.52" + GH_AW_WORKFLOW_EMOJI: "🔧" GH_AW_WORKFLOW_ID: "issue-triage-agent" GH_AW_WORKFLOW_NAME: "Issue Triage Agent" - GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/tree/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github/workflows/issue-triage-agent.md" + GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/blob/359795d49ada21681ab616bd4cbcb144a7387115/.github/workflows/issue-triage-agent.md" outputs: + code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }} + code_push_failure_errors: ${{ steps.process_safe_outputs.outputs.code_push_failure_errors }} + comment_id: ${{ steps.process_safe_outputs.outputs.comment_id }} + comment_url: ${{ steps.process_safe_outputs.outputs.comment_url }} create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }} create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }} process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@0feed75a980b06f247abbbf80127f8eb2c19e2c5 # v0.74.8 + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/actions + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage-agent.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" - name: Download agent output artifact + id: download-agent-output continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent-output - path: /tmp/gh-aw/safeoutputs/ + name: agent + path: /tmp/gh-aw/ - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Configure GH_HOST for enterprise compatibility + id: ghes-host-config + shell: bash run: | - mkdir -p /tmp/gh-aw/safeoutputs/ - find "/tmp/gh-aw/safeoutputs/" -type f -print - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" + # Derive GH_HOST from GITHUB_SERVER_URL so the gh CLI targets the correct + # GitHub instance (GHES/GHEC). On github.com this is a harmless no-op. + GH_HOST="${GITHUB_SERVER_URL#https://}" + GH_HOST="${GH_HOST#http://}" + echo "GH_HOST=${GH_HOST}" >> "$GITHUB_ENV" - name: Process Safe Outputs id: process_safe_outputs uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":1},\"add_labels\":{\"allowed\":[\"bug\",\"feature\",\"enhancement\",\"documentation\",\"question\",\"help-wanted\",\"good-first-issue\"]},\"missing_data\":{},\"missing_tool\":{}}" + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }} + GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":1},\"add_labels\":{\"allowed\":[\"bug\",\"feature\",\"enhancement\",\"documentation\",\"question\",\"help-wanted\",\"good-first-issue\"]},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/safe_output_handler_manager.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/safe_output_handler_manager.cjs'); await main(); + - name: Upload Safe Outputs Items + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: safe-outputs-items + path: | + /tmp/gh-aw/safe-output-items.jsonl + /tmp/gh-aw/temporary-id-map.json + if-no-files-found: ignore diff --git a/.github/workflows/issue-triage-agent.md b/.github/workflows/issue-triage-agent.md index 06ff227b740d..26c43644618c 100644 --- a/.github/workflows/issue-triage-agent.md +++ b/.github/workflows/issue-triage-agent.md @@ -1,11 +1,11 @@ --- on: - schedule: 0 14 * * 1-5 + schedule: daily around 14:00 on weekdays workflow_dispatch: null permissions: issues: read imports: -- github/gh-aw/.github/workflows/shared/reporting.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4 +- github/gh-aw/.github/workflows/shared/reporting.md@359795d49ada21681ab616bd4cbcb144a7387115 safe-outputs: add-comment: {} add-labels: @@ -17,10 +17,12 @@ safe-outputs: - question - help-wanted - good-first-issue -source: github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4 +emoji: 🔧 +source: github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115 strict: true timeout-minutes: 5 tools: + cli-proxy: true github: toolsets: - issues @@ -45,7 +47,7 @@ Hi @{author}! I've categorized this issue as **{label_name}** based on the follo **Reasoning**: {brief_explanation_of_why_this_label}
-View Triage Details +View Triage Details #### Analysis - **Keywords detected**: {list_of_keywords_that_matched} @@ -76,3 +78,15 @@ For efficiency, if multiple issues are triaged in a single run: 3. Optionally: Create a discussion summarizing all triage actions for that run This provides both per-issue context and batch visibility. + +## Labels + +- `bug`: Indicates a problem or error in the code that needs fixing. +- `feature`: Represents a new feature request or enhancement to existing functionality. +- `enhancement`: Suggests improvements to existing features or code. +- `documentation`: Pertains to issues related to documentation, such as missing or unclear docs. +- `question`: Used for issues that are asking for clarification or have questions about the project. +- `help-wanted`: Indicates that the issue is a good candidate for external contributions and help +- `good-first-issue`: Marks issues that are suitable for newcomers to the project, often with simpler scope. + +{{#runtime-import shared/noop-reminder.md}} From be5194814616ec8d3bf9410ed6bc533186bd0eed Mon Sep 17 00:00:00 2001 From: Gabriel Pordeus Santos Date: Wed, 3 Jun 2026 15:06:06 -0300 Subject: [PATCH 11/28] Stop role from auto-changing if user manually selected a role during account creation (#13320) --- ui/src/views/iam/AddAccount.vue | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ui/src/views/iam/AddAccount.vue b/ui/src/views/iam/AddAccount.vue index 25b45cded4ec..a211fd2d8c3d 100644 --- a/ui/src/views/iam/AddAccount.vue +++ b/ui/src/views/iam/AddAccount.vue @@ -36,6 +36,7 @@ v-focus="true" showSearch optionFilterProp="label" + @change="onRoleChange" :filterOption="(input, option) => { return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0 }"> @@ -211,6 +212,7 @@ export default { domain: { id: null, loading: false }, domainsList: [], dom: null, + roleManuallySelected: false, roleLoading: false, roles: [], timeZoneLoading: false, @@ -299,8 +301,13 @@ export default { }) } }, + onRoleChange (newRoleId) { + if (newRoleId) { + this.roleManuallySelected = true + } + }, setDefaultRole () { - if (this.roles.length === 0) return + if (this.roles.length === 0 || this.roleManuallySelected) return let targetRoleType = null From a1959f2dc2e93badc5f3e04a85922f9d391c8ba0 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 8 Jun 2026 12:17:00 +0530 Subject: [PATCH 12/28] backup: veeam kvm integration (#12991) This PR introduces the initial implementation of Veeam integration support for KVM in CloudStack by adding a UHAPI-compatible server and image server components. Veeam Backup & Replication interacts with virtualization platforms using its Universal Hypervisor API (UHAPI). To enable backup and restore workflows for CloudStack-managed KVM environments, this change introduces a UHAPI server that exposes CloudStack resources through a UHAPI-compatible interface. In addition to the control plane APIs, an image server component is introduced to handle the data transfer operations required during backup and restore workflows. The integration consists of two main components: 1. UHAPI Server (Control Plane) named CloudStack Veeam Control Service A lightweight UHAPI server runs inside the CloudStack management server and exposes endpoints under: /ovirt-engine - /api - For APIs - /sso - For authentication - /services/pki-resource - For certificates This server provides inventory discovery APIs required by Veeam and translates CloudStack resources into the structures expected by UHAPI. The server: - exposes infrastructure inventory - handles authentication and session tokens - maps CloudStack resources to UHAPI-compatible representations 2. Image Server (Data Plane) named CloudStack Image Service A separate image server component is introduced to handle backup and restore data transfer operations. This component: - serves disk image data during backup - receives image data during restore operations - exposes endpoints used by Veeam worker components - integrates with CloudStack storage to read and write VM disk data The separation between both these components server ensures that: - metadata APIs and control operations remain lightweight - bulk image transfer operations are handled independently Signed-off-by: Abhishek Kumar Co-authored-by: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Co-authored-by: abh1sar Co-authored-by: Wei Zhou --- agent/conf/agent.properties | 8 + .../agent/properties/AgentProperties.java | 14 + .../com/cloud/storage/VolumeApiService.java | 9 + .../java/com/cloud/user/AccountService.java | 6 + .../java/com/cloud/vm/VmDetailConstants.java | 6 + .../apache/cloudstack/api/ApiConstants.java | 5 + .../cloudstack/api/ApiServerService.java | 19 + .../api/BaseAsyncCreateCustomIdCmd.java | 2 +- .../admin/backup/CreateImageTransferCmd.java | 100 + .../admin/backup/DeleteVmCheckpointCmd.java | 85 + .../admin/backup/FinalizeBackupCmd.java | 103 + .../backup/FinalizeImageTransferCmd.java | 69 + .../admin/backup/ListImageTransfersCmd.java | 81 + .../admin/backup/ListVmCheckpointsCmd.java | 69 + .../command/admin/backup/StartBackupCmd.java | 120 + .../command/admin/config/ListCfgsByCmd.java | 24 +- .../api/command/admin/config/ResetCfgCmd.java | 20 +- .../command/admin/config/UpdateCfgCmd.java | 24 +- .../api/command/admin/vm/AssignVMCmd.java | 31 + .../command/admin/vm/DeployVMCmdByAdmin.java | 46 + .../command/admin/vm/DestroyVMCmdByAdmin.java | 20 +- .../offering/ListServiceOfferingsCmd.java | 16 + .../api/command/user/vm/AddNicToVMCmd.java | 20 + .../api/command/user/vm/BaseDeployVMCmd.java | 45 +- .../api/command/user/vm/DeployVMCmd.java | 111 +- .../api/command/user/vm/DestroyVMCmd.java | 4 + .../api/command/user/vm/UpdateVMCmd.java | 8 + .../command/user/volume/AssignVolumeCmd.java | 15 + .../command/user/volume/CreateVolumeCmd.java | 10 +- .../command/user/volume/DetachVolumeCmd.java | 4 + .../api/response/BackupResponse.java | 36 + .../api/response/CheckpointResponse.java | 53 + .../api/response/ImageTransferResponse.java | 104 + .../org/apache/cloudstack/backup/Backup.java | 10 +- .../cloudstack/backup/ImageTransfer.java | 61 + .../backup/KVMBackupExportService.java | 106 + .../storage/sharedfs/SharedFSService.java | 15 +- .../api/command/admin/vm/AssignVMCmdTest.java | 55 + .../admin/vm/DeployVMCmdByAdminTest.java | 75 + .../api/command/user/vm/DeployVMCmdTest.java | 144 ++ client/pom.xml | 5 + .../backup/CreateImageTransferAnswer.java | 56 + .../backup/CreateImageTransferCommand.java | 94 + .../backup/DeleteVmCheckpointCommand.java | 60 + .../backup/FinalizeImageTransferCommand.java | 40 + .../cloudstack/backup/StartBackupAnswer.java | 44 + .../cloudstack/backup/StartBackupCommand.java | 91 + .../backup/StartNBDServerAnswer.java | 56 + .../backup/StartNBDServerCommand.java | 78 + .../cloudstack/backup/StopBackupAnswer.java | 30 + .../cloudstack/backup/StopBackupCommand.java | 52 + .../backup/StopNBDServerCommand.java | 46 + debian/control | 2 +- .../com/cloud/vm/VirtualMachineManager.java | 6 + .../cloud/vm/VirtualMachineManagerImpl.java | 24 +- .../orchestration/CloudOrchestrator.java | 7 +- ...ring-engine-orchestration-core-context.xml | 1 + .../java/com/cloud/dc/dao/ClusterDao.java | 3 + .../java/com/cloud/dc/dao/ClusterDaoImpl.java | 18 + .../com/cloud/network/dao/NetworkDao.java | 6 + .../com/cloud/network/dao/NetworkDaoImpl.java | 42 +- .../security/dao/SecurityGroupDao.java | 2 + .../security/dao/SecurityGroupDaoImpl.java | 10 + .../cloud/storage/dao/DiskOfferingDao.java | 2 +- .../storage/dao/DiskOfferingDaoImpl.java | 8 +- .../com/cloud/storage/dao/VMTemplateDao.java | 2 + .../cloud/storage/dao/VMTemplateDaoImpl.java | 8 + .../java/com/cloud/storage/dao/VolumeDao.java | 2 + .../com/cloud/storage/dao/VolumeDaoImpl.java | 11 + .../com/cloud/tags/dao/ResourceTagDao.java | 13 +- .../cloud/tags/dao/ResourceTagsDaoImpl.java | 67 +- .../main/java/com/cloud/vm/dao/NicDao.java | 2 + .../java/com/cloud/vm/dao/NicDaoImpl.java | 7 + .../java/com/cloud/vm/dao/VMInstanceDao.java | 2 + .../com/cloud/vm/dao/VMInstanceDaoImpl.java | 16 + .../apache/cloudstack/backup/BackupVO.java | 48 + .../cloudstack/backup/ImageTransferVO.java | 242 ++ .../backup/dao/ImageTransferDao.java | 36 + .../backup/dao/ImageTransferDaoImpl.java | 136 + .../storage/sharedfs/dao/SharedFSDao.java | 2 + .../storage/sharedfs/dao/SharedFSDaoImpl.java | 10 + ...spring-engine-schema-core-daos-context.xml | 2 + .../META-INF/db/schema-42100to42200.sql | 1 - .../META-INF/db/schema-42210to42300.sql | 47 + .../META-INF/db/views/cloud.user_vm_view.sql | 1 + .../cloud/storage/dao/VolumeDaoImplTest.java | 65 +- .../userdata/UserDataManagerImpl.java | 6 +- framework/cluster/pom.xml | 6 + .../cluster/ManagementServerHostDetailVO.java | 87 + .../dao/ManagementServerHostDetailsDao.java | 26 + .../ManagementServerHostDetailsDaoImpl.java | 46 + .../framework/jobs/dao/AsyncJobDao.java | 2 + .../framework/jobs/dao/AsyncJobDaoImpl.java | 10 + packaging/el8/cloud.spec | 2 + .../resource/ImageServerControlSocket.java | 123 + .../resource/LibvirtComputingResource.java | 45 + ...virtCreateImageTransferCommandWrapper.java | 178 ++ ...bvirtDeleteVmCheckpointCommandWrapper.java | 79 + ...rtFinalizeImageTransferCommandWrapper.java | 101 + .../LibvirtStartBackupCommandWrapper.java | 300 +++ .../LibvirtStartNBDServerCommandWrapper.java | 197 ++ .../LibvirtStopBackupCommandWrapper.java | 69 + .../LibvirtStopNBDServerCommandWrapper.java | 72 + ...CreateImageTransferCommandWrapperTest.java | 121 + ...tDeleteVmCheckpointCommandWrapperTest.java | 88 + ...nalizeImageTransferCommandWrapperTest.java | 87 + .../LibvirtStartBackupCommandWrapperTest.java | 182 ++ ...bvirtStartNBDServerCommandWrapperTest.java | 179 ++ .../LibvirtStopBackupCommandWrapperTest.java | 95 + .../veeam-control-service/pom.xml | 66 + .../apache/cloudstack/veeam/RouteHandler.java | 81 + .../cloudstack/veeam/VeeamControlServer.java | 221 ++ .../cloudstack/veeam/VeeamControlService.java | 89 + .../veeam/VeeamControlServiceImpl.java | 157 ++ .../cloudstack/veeam/VeeamControlServlet.java | 161 ++ .../cloudstack/veeam/adapter/ApiAccess.java | 31 + .../veeam/adapter/ApiAccessInterceptor.java | 68 + .../veeam/adapter/ServerAdapter.java | 2188 +++++++++++++++++ .../cloudstack/veeam/api/ApiRouteHandler.java | 129 + .../veeam/api/ClustersRouteHandler.java | 110 + .../veeam/api/DataCentersRouteHandler.java | 151 ++ .../veeam/api/DisksRouteHandler.java | 201 ++ .../veeam/api/HostsRouteHandler.java | 110 + .../veeam/api/ImageTransfersRouteHandler.java | 162 ++ .../veeam/api/JobsRouteHandler.java | 108 + .../veeam/api/NetworksRouteHandler.java | 110 + .../veeam/api/TagsRouteHandler.java | 111 + .../cloudstack/veeam/api/VmsRouteHandler.java | 580 +++++ .../veeam/api/VnicProfilesRouteHandler.java | 110 + .../AsyncJobJoinVOToJobConverter.java | 90 + .../converter/BackupVOToBackupConverter.java | 99 + .../ClusterVOToClusterConverter.java | 155 ++ ...DataCenterJoinVOToDataCenterConverter.java | 76 + .../converter/HostJoinVOToHostConverter.java | 103 + ...ageTransferVOToImageTransferConverter.java | 89 + .../NetworkVOToNetworkConverter.java | 79 + .../NetworkVOToVnicProfileConverter.java | 65 + .../api/converter/NicVOToNicConverter.java | 116 + .../ResourceTagVOToTagConverter.java | 82 + .../StoreVOToStorageDomainConverter.java | 244 ++ .../converter/UserVmJoinVOToVmConverter.java | 270 ++ .../UserVmVOToCheckpointConverter.java | 45 + .../VmSnapshotVOToSnapshotConverter.java | 54 + .../VolumeJoinVOToDiskConverter.java | 198 ++ .../apache/cloudstack/veeam/api/dto/Api.java | 104 + .../cloudstack/veeam/api/dto/ApiSummary.java | 61 + .../cloudstack/veeam/api/dto/Backup.java | 103 + .../cloudstack/veeam/api/dto/BaseDto.java | 55 + .../cloudstack/veeam/api/dto/Certificate.java | 42 + .../cloudstack/veeam/api/dto/Checkpoint.java | 76 + .../cloudstack/veeam/api/dto/Cluster.java | 408 +++ .../apache/cloudstack/veeam/api/dto/Cpu.java | 69 + .../cloudstack/veeam/api/dto/DataCenter.java | 129 + .../apache/cloudstack/veeam/api/dto/Disk.java | 247 ++ .../veeam/api/dto/DiskAttachment.java | 124 + .../veeam/api/dto/EmptyElement.java | 26 + .../veeam/api/dto/EmptyElementSerializer.java | 36 + .../cloudstack/veeam/api/dto/Fault.java | 39 + .../apache/cloudstack/veeam/api/dto/Host.java | 315 +++ .../veeam/api/dto/ImageTransfer.java | 167 ++ .../apache/cloudstack/veeam/api/dto/Ip.java | 61 + .../apache/cloudstack/veeam/api/dto/Job.java | 117 + .../apache/cloudstack/veeam/api/dto/Link.java | 49 + .../apache/cloudstack/veeam/api/dto/Mac.java | 34 + .../cloudstack/veeam/api/dto/NamedList.java | 63 + .../cloudstack/veeam/api/dto/Network.java | 126 + .../apache/cloudstack/veeam/api/dto/Nic.java | 122 + .../apache/cloudstack/veeam/api/dto/Os.java | 114 + .../cloudstack/veeam/api/dto/OvfXmlUtil.java | 1138 +++++++++ .../cloudstack/veeam/api/dto/ProductInfo.java | 52 + .../apache/cloudstack/veeam/api/dto/Ref.java | 31 + .../veeam/api/dto/ReportedDevice.java | 84 + .../veeam/api/dto/ResourceAction.java | 39 + .../cloudstack/veeam/api/dto/Snapshot.java | 105 + .../veeam/api/dto/SpecialObjects.java | 43 + .../cloudstack/veeam/api/dto/Storage.java | 70 + .../veeam/api/dto/StorageDomain.java | 236 ++ .../veeam/api/dto/SummaryCount.java | 43 + .../veeam/api/dto/SupportedVersions.java | 38 + .../apache/cloudstack/veeam/api/dto/Tag.java | 57 + .../cloudstack/veeam/api/dto/Topology.java | 60 + .../cloudstack/veeam/api/dto/Version.java | 94 + .../apache/cloudstack/veeam/api/dto/Vm.java | 643 +++++ .../cloudstack/veeam/api/dto/VmAction.java | 30 + .../cloudstack/veeam/api/dto/VnicProfile.java | 81 + .../veeam/api/request/ListQuery.java | 158 ++ .../filter/AllowedClientCidrsFilter.java | 100 + .../veeam/filter/BearerOrBasicAuthFilter.java | 220 ++ .../services/PkiResourceRouteHandler.java | 106 + .../cloudstack/veeam/sso/SsoService.java | 136 + .../veeam/utils/CloudConfigUtil.java | 156 ++ .../cloudstack/veeam/utils/DataUtil.java | 44 + .../cloudstack/veeam/utils/JwtUtil.java | 61 + .../apache/cloudstack/veeam/utils/Mapper.java | 63 + .../cloudstack/veeam/utils/Negotiation.java | 45 + .../cloudstack/veeam/utils/PathUtil.java | 75 + .../veeam/utils/ResponseWriter.java | 87 + .../veeam-control-service/module.properties | 18 + .../spring-veeam-control-service-context.xml | 66 + .../veeam/VeeamControlServerTest.java | 227 ++ .../veeam/VeeamControlServiceImplTest.java | 185 ++ .../veeam/VeeamControlServletTest.java | 121 + .../adapter/ApiAccessInterceptorTest.java | 204 ++ .../veeam/adapter/ServerAdapterTest.java | 1140 +++++++++ .../veeam/api/ApiRouteHandlerTest.java | 91 + .../veeam/api/ClustersRouteHandlerTest.java | 80 + .../api/DataCentersRouteHandlerTest.java | 91 + .../veeam/api/DisksRouteHandlerTest.java | 122 + .../veeam/api/HostsRouteHandlerTest.java | 64 + .../api/ImageTransfersRouteHandlerTest.java | 102 + .../veeam/api/JobsRouteHandlerTest.java | 67 + .../veeam/api/NetworksRouteHandlerTest.java | 63 + .../veeam/api/RouteHandlerTestSupport.java | 102 + .../veeam/api/TagsRouteHandlerTest.java | 63 + .../veeam/api/VmsRouteHandlerTest.java | 297 +++ .../api/VnicProfilesRouteHandlerTest.java | 63 + .../AsyncJobJoinVOToJobConverterTest.java | 85 + .../BackupVOToBackupConverterTest.java | 99 + .../ClusterVOToClusterConverterTest.java | 81 + ...CenterJoinVOToDataCenterConverterTest.java | 88 + .../HostJoinVOToHostConverterTest.java | 123 + ...ransferVOToImageTransferConverterTest.java | 41 + .../NetworkVOToNetworkConverterTest.java | 113 + .../NetworkVOToVnicProfileConverterTest.java | 100 + .../converter/NicVOToNicConverterTest.java | 66 + .../ResourceTagVOToTagConverterTest.java | 87 + .../StoreVOToStorageDomainConverterTest.java | 99 + .../UserVmJoinVOToVmConverterTest.java | 140 ++ .../UserVmVOToCheckpointConverterTest.java | 56 + .../VmSnapshotVOToSnapshotConverterTest.java | 74 + .../VolumeJoinVOToDiskConverterTest.java | 100 + .../cloudstack/veeam/api/dto/BackupTest.java | 79 + .../cloudstack/veeam/api/dto/BaseDtoTest.java | 100 + .../veeam/api/dto/CertificateTest.java | 45 + .../cloudstack/veeam/api/dto/DiskTest.java | 68 + .../veeam/api/dto/DtoSerializationTest.java | 82 + .../cloudstack/veeam/api/dto/FaultTest.java | 41 + .../cloudstack/veeam/api/dto/HostTest.java | 73 + .../veeam/api/dto/ImageTransferTest.java | 67 + .../cloudstack/veeam/api/dto/MacIpTest.java | 67 + .../veeam/api/dto/NamedListTest.java | 93 + .../veeam/api/dto/OvfXmlUtilTest.java | 81 + .../veeam/api/dto/ProductInfoApiTest.java | 95 + .../veeam/api/dto/SnapshotTest.java | 51 + .../veeam/api/dto/StorageDomainTest.java | 53 + .../veeam/api/dto/SummaryCountTest.java | 57 + .../veeam/api/dto/TopologyTest.java | 78 + .../cloudstack/veeam/api/dto/VersionTest.java | 80 + .../cloudstack/veeam/api/dto/VmTest.java | 98 + .../dto/VnicProfileReportedDeviceTest.java | 83 + .../veeam/api/request/ListQueryTest.java | 92 + .../filter/AllowedClientCidrsFilterTest.java | 130 + .../filter/BearerOrBasicAuthFilterTest.java | 129 + .../services/PkiResourceRouteHandlerTest.java | 127 + .../cloudstack/veeam/sso/SsoServiceTest.java | 219 ++ .../veeam/utils/CloudConfigUtilTest.java | 82 + .../cloudstack/veeam/utils/DataUtilTest.java | 57 + .../cloudstack/veeam/utils/JwtUtilTest.java | 79 + .../cloudstack/veeam/utils/MapperTest.java | 72 + .../veeam/utils/NegotiationTest.java | 66 + .../cloudstack/veeam/utils/PathUtilTest.java | 54 + .../veeam/utils/ResponseWriterTest.java | 116 + .../src/test/resources/test-ovf.xml | 225 ++ .../management/MockAccountManager.java | 40 +- plugins/pom.xml | 1 + .../vm/hypervisor/kvm/imageserver/__init__.py | 32 + .../vm/hypervisor/kvm/imageserver/__main__.py | 20 + .../kvm/imageserver/backends/__init__.py | 36 + .../kvm/imageserver/backends/base.py | 148 ++ .../kvm/imageserver/backends/file.py | 123 + .../kvm/imageserver/backends/nbd.py | 471 ++++ .../vm/hypervisor/kvm/imageserver/config.py | 204 ++ .../hypervisor/kvm/imageserver/constants.py | 55 + .../vm/hypervisor/kvm/imageserver/handler.py | 899 +++++++ .../vm/hypervisor/kvm/imageserver/server.py | 222 ++ .../kvm/imageserver/tests/__init__.py | 22 + .../kvm/imageserver/tests/test_base.py | 568 +++++ .../imageserver/tests/test_combinations.py | 397 +++ .../imageserver/tests/test_control_socket.py | 258 ++ .../imageserver/tests/test_file_backend.py | 230 ++ .../tests/test_http11_lifecycle.py | 148 ++ .../kvm/imageserver/tests/test_nbd_backend.py | 706 ++++++ .../imageserver/tests/test_registry_idle.py | 101 + .../kvm/imageserver/tests/test_stress_io.py | 412 ++++ .../tests/test_transfer_idle_expiry.py | 57 + .../kvm/imageserver/tests/test_util.py | 122 + scripts/vm/hypervisor/kvm/imageserver/util.py | 125 + .../main/java/com/cloud/api/ApiServer.java | 188 +- .../cloud/api/query/dao/AsyncJobJoinDao.java | 4 + .../api/query/dao/AsyncJobJoinDaoImpl.java | 18 +- .../api/query/dao/DataCenterJoinDao.java | 5 + .../api/query/dao/DataCenterJoinDaoImpl.java | 33 +- .../com/cloud/api/query/dao/HostJoinDao.java | 4 + .../cloud/api/query/dao/HostJoinDaoImpl.java | 26 + .../api/query/dao/StoragePoolJoinDao.java | 5 + .../api/query/dao/StoragePoolJoinDaoImpl.java | 31 +- .../cloud/api/query/dao/UserVmJoinDao.java | 6 + .../api/query/dao/UserVmJoinDaoImpl.java | 43 +- .../cloud/api/query/dao/VolumeJoinDao.java | 7 + .../api/query/dao/VolumeJoinDaoImpl.java | 49 +- .../com/cloud/api/query/vo/UserVmJoinVO.java | 13 +- .../com/cloud/api/query/vo/VolumeJoinVO.java | 5 + .../ConfigurationManagerImpl.java | 34 +- .../cloud/server/ManagementServerImpl.java | 6 + .../cloud/storage/VolumeApiServiceImpl.java | 131 +- .../java/com/cloud/user/AccountManager.java | 2 + .../com/cloud/user/AccountManagerImpl.java | 25 + .../main/java/com/cloud/vm/UserVmManager.java | 7 +- .../java/com/cloud/vm/UserVmManagerImpl.java | 188 +- .../cloudstack/backup/BackupManagerImpl.java | 19 +- .../backup/KVMBackupExportServiceImpl.java | 1036 ++++++++ .../backup/VmWorkWaitForBackupFinalize.java | 35 + .../storage/sharedfs/SharedFSServiceImpl.java | 33 + .../spring-server-core-managers-context.xml | 2 + .../query/dao/StoragePoolJoinDaoImplTest.java | 121 + .../ConfigurationManagerImplTest.java | 1 + .../server/ManagementServerImplTest.java | 1 + .../storage/VolumeApiServiceImplTest.java | 139 +- .../com/cloud/vm/UserVmManagerImplTest.java | 5 +- .../com/cloud/vpc/dao/MockNetworkDaoImpl.java | 22 +- .../cloudstack/backup/BackupManagerTest.java | 40 +- .../KVMBackupExportServiceImplTest.java | 467 ++++ .../resource/NfsSecondaryStorageResource.java | 4 +- tools/apidoc/gen_toc.py | 4 + ui/src/components/view/InfoCard.vue | 12 +- ui/src/components/view/SettingsTab.vue | 3 + .../config/section/infra/managementServers.js | 4 + ui/src/utils/links.js | 72 + .../utils/server/ServerPropertiesUtil.java | 11 + 329 files changed, 34111 insertions(+), 352 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java create mode 100644 api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java create mode 100644 api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java create mode 100644 api/src/test/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmdTest.java create mode 100644 api/src/test/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdminTest.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferAnswer.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/DeleteVmCheckpointCommand.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/StartNBDServerAnswer.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/StopBackupAnswer.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/StopBackupCommand.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/StopNBDServerCommand.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java create mode 100644 framework/cluster/src/main/java/com/cloud/cluster/ManagementServerHostDetailVO.java create mode 100644 framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDao.java create mode 100644 framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDaoImpl.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/ImageServerControlSocket.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapper.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopBackupCommandWrapper.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopNBDServerCommandWrapper.java create mode 100644 plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapperTest.java create mode 100644 plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapperTest.java create mode 100644 plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapperTest.java create mode 100644 plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapperTest.java create mode 100644 plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapperTest.java create mode 100644 plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopBackupCommandWrapperTest.java create mode 100644 plugins/integrations/veeam-control-service/pom.xml create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/RouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServer.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlService.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServiceImpl.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/VeeamControlServlet.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ApiAccess.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ApiAccessInterceptor.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/adapter/ServerAdapter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ApiRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ClustersRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/DisksRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/HostsRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/JobsRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/NetworksRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/TagsRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VmsRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/BackupVOToBackupConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/DataCenterJoinVOToDataCenterConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/HostJoinVOToHostConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToNetworkConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToVnicProfileConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/NicVOToNicConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/ResourceTagVOToTagConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/StoreVOToStorageDomainConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/UserVmVOToCheckpointConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VmSnapshotVOToSnapshotConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Api.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ApiSummary.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Backup.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/BaseDto.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Certificate.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Checkpoint.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cluster.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Cpu.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DataCenter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Disk.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/DiskAttachment.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/EmptyElement.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/EmptyElementSerializer.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Fault.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Host.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ImageTransfer.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ip.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Job.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Link.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Mac.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/NamedList.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Network.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Nic.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Os.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtil.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ProductInfo.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Ref.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ReportedDevice.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/ResourceAction.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Snapshot.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SpecialObjects.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Storage.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/StorageDomain.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SummaryCount.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/SupportedVersions.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Tag.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Topology.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Version.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/Vm.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VmAction.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/dto/VnicProfile.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/api/request/ListQuery.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/AllowedClientCidrsFilter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/filter/BearerOrBasicAuthFilter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/services/PkiResourceRouteHandler.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/sso/SsoService.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/CloudConfigUtil.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/DataUtil.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/JwtUtil.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/Mapper.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/Negotiation.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/PathUtil.java create mode 100644 plugins/integrations/veeam-control-service/src/main/java/org/apache/cloudstack/veeam/utils/ResponseWriter.java create mode 100644 plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/module.properties create mode 100644 plugins/integrations/veeam-control-service/src/main/resources/META-INF/cloudstack/veeam-control-service/spring-veeam-control-service-context.xml create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/VeeamControlServerTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/VeeamControlServiceImplTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/VeeamControlServletTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/adapter/ApiAccessInterceptorTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/adapter/ServerAdapterTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/ApiRouteHandlerTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/ClustersRouteHandlerTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/DataCentersRouteHandlerTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/DisksRouteHandlerTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/HostsRouteHandlerTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/ImageTransfersRouteHandlerTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/JobsRouteHandlerTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/NetworksRouteHandlerTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/RouteHandlerTestSupport.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/TagsRouteHandlerTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/VmsRouteHandlerTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/VnicProfilesRouteHandlerTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/converter/AsyncJobJoinVOToJobConverterTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/converter/BackupVOToBackupConverterTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/converter/ClusterVOToClusterConverterTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/converter/DataCenterJoinVOToDataCenterConverterTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/converter/HostJoinVOToHostConverterTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/converter/ImageTransferVOToImageTransferConverterTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToNetworkConverterTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/converter/NetworkVOToVnicProfileConverterTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/converter/NicVOToNicConverterTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/converter/ResourceTagVOToTagConverterTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/converter/StoreVOToStorageDomainConverterTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/converter/UserVmJoinVOToVmConverterTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/converter/UserVmVOToCheckpointConverterTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/converter/VmSnapshotVOToSnapshotConverterTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/converter/VolumeJoinVOToDiskConverterTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/BackupTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/BaseDtoTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/CertificateTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/DiskTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/DtoSerializationTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/FaultTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/HostTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/ImageTransferTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/MacIpTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/NamedListTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/OvfXmlUtilTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/ProductInfoApiTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/SnapshotTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/StorageDomainTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/SummaryCountTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/TopologyTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/VersionTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/VmTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/dto/VnicProfileReportedDeviceTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/api/request/ListQueryTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/filter/AllowedClientCidrsFilterTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/filter/BearerOrBasicAuthFilterTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/services/PkiResourceRouteHandlerTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/sso/SsoServiceTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/utils/CloudConfigUtilTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/utils/DataUtilTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/utils/JwtUtilTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/utils/MapperTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/utils/NegotiationTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/utils/PathUtilTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/java/org/apache/cloudstack/veeam/utils/ResponseWriterTest.java create mode 100644 plugins/integrations/veeam-control-service/src/test/resources/test-ovf.xml create mode 100644 scripts/vm/hypervisor/kvm/imageserver/__init__.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/__main__.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/backends/__init__.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/backends/base.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/backends/file.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/backends/nbd.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/config.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/constants.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/handler.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/server.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/__init__.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/test_base.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/test_combinations.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/test_control_socket.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/test_file_backend.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/test_http11_lifecycle.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/test_nbd_backend.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/test_registry_idle.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/test_stress_io.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/test_transfer_idle_expiry.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/tests/test_util.py create mode 100644 scripts/vm/hypervisor/kvm/imageserver/util.py create mode 100644 server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java create mode 100644 server/src/main/java/org/apache/cloudstack/backup/VmWorkWaitForBackupFinalize.java create mode 100644 server/src/test/java/com/cloud/api/query/dao/StoragePoolJoinDaoImplTest.java create mode 100644 server/src/test/java/org/apache/cloudstack/backup/KVMBackupExportServiceImplTest.java diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index ba4a3874664a..0a459d8d4fc5 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -78,6 +78,14 @@ zone=default # Generated with "uuidgen". local.storage.uuid= +# Enable TLS for image server transfers. The keys are read from: +# cert file = /etc/cloudstack/agent/cloud.crt +# key file = /etc/cloudstack/agent/cloud.key +image.server.tls.enabled=true + +# The Address for the network interface that the image server listens on. If not specified, it will listen on the Management network. +#image.server.listen.address= + # Location for KVM virtual router scripts. # The path defined in this property is relative to the directory "/usr/share/cloudstack-common/". domr.scripts.dir=scripts/network/domr/kvm diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index e69a7efdc9c7..18a6b6df1006 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -123,6 +123,20 @@ public class AgentProperties{ */ public static final Property LOCAL_STORAGE_PATH = new Property<>("local.storage.path", "/var/lib/libvirt/images/"); + /** + * Enables TLS on the KVM image server transfer endpoint.
+ * Data type: Boolean.
+ * Default value: true + */ + public static final Property IMAGE_SERVER_TLS_ENABLED = new Property<>("image.server.tls.enabled", true); + + /** + * The IP address that the KVM image server listens on.
+ * Data type: String.
+ * Default value: null + */ + public static final Property IMAGE_SERVER_LISTEN_ADDRESS = new Property<>("image.server.listen.address", null, String.class); + /** * Directory where Qemu sockets are placed.
* These sockets are for the Qemu Guest Agent and SSVM provisioning.
diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index 1a9bcc6ee98b..43ead8d287b9 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; +import com.cloud.dc.DataCenter; import com.cloud.exception.ResourceAllocationException; import com.cloud.offering.DiskOffering; import com.cloud.user.Account; @@ -70,6 +71,10 @@ public interface VolumeApiService { */ Volume allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationException; + Volume allocVolume(long ownerId, Long zoneId, Long diskOfferingId, Long vmId, Long snapshotId, String name, + Long cmdSize, Boolean displayVolume, Long cmdMinIops, Long cmdMaxIops, String customId) + throws ResourceAllocationException; + /** * Creates the volume based on the given criteria * @@ -80,6 +85,8 @@ public interface VolumeApiService { */ Volume createVolume(CreateVolumeCmd cmd); + Volume createVolume(long volumeId, Long vmId, Long snapshotId, Long storageId, Boolean display); + /** * Resizes the volume based on the given criteria * @@ -203,4 +210,6 @@ Volume updateVolume(long volumeId, String path, String state, Long storageId, Pair checkAndRepairVolume(CheckAndRepairVolumeCmd cmd) throws ResourceAllocationException; Long getVolumePhysicalSize(Storage.ImageFormat format, String path, String chainInfo); + + Long getCustomDiskOfferingIdForVolumeUpload(Account owner, DataCenter zone, boolean encryptEnabledOnly); } diff --git a/api/src/main/java/com/cloud/user/AccountService.java b/api/src/main/java/com/cloud/user/AccountService.java index 4145e2b89eb3..fc450e9179c5 100644 --- a/api/src/main/java/com/cloud/user/AccountService.java +++ b/api/src/main/java/com/cloud/user/AccountService.java @@ -88,10 +88,16 @@ User createUser(String userName, String password, String firstName, String lastN Account getActiveAccountById(long accountId); + Account getActiveAccountByUuid(String accountUuid); + Account getAccount(long accountId); + Account getAccountByUuid(String accountUuid); + User getActiveUser(long userId); + User getOneActiveUserForAccount(Account account); + User getUserIncludingRemoved(long userId); boolean isRootAdmin(Long accountId); diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index 9e56bf4f17b2..33cc6da70812 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -130,4 +130,10 @@ public interface VmDetailConstants { String EXTERNAL_DETAIL_PREFIX = "External:"; String CLOUDSTACK_VM_DETAILS = "cloudstack.vm.details"; String CLOUDSTACK_VLAN = "cloudstack.vlan"; + + // KVM Checkpoints related + String ACTIVE_CHECKPOINT_ID = "active.checkpoint.id"; + String ACTIVE_CHECKPOINT_CREATE_TIME = "active.checkpoint.create.time"; + String LAST_CHECKPOINT_ID = "last.checkpoint.id"; + String LAST_CHECKPOINT_CREATE_TIME = "last.checkpoint.create.time"; } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 694830ea2f36..ee10db031ba7 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -79,6 +79,7 @@ public class ApiConstants { public static final String BOOTABLE = "bootable"; public static final String BIND_DN = "binddn"; public static final String BIND_PASSWORD = "bindpass"; + public static final String BLANK_INSTANCE = "blankinstance"; public static final String BUS_ADDRESS = "busaddress"; public static final String BYTES_READ_RATE = "bytesreadrate"; public static final String BYTES_READ_RATE_MAX = "bytesreadratemax"; @@ -220,6 +221,7 @@ public class ApiConstants { public static final String DOMAIN_PATH = "domainpath"; public static final String DOMAIN_ID = "domainid"; public static final String DOMAIN__ID = "domainId"; + public static final String DUMMY = "dummy"; public static final String DURATION = "duration"; public static final String ELIGIBLE = "eligible"; public static final String EMAIL = "email"; @@ -263,6 +265,7 @@ public class ApiConstants { public static final String FOR_VIRTUAL_NETWORK = "forvirtualnetwork"; public static final String FOR_SYSTEM_VMS = "forsystemvms"; public static final String FOR_PROVIDER = "forprovider"; + public static final String FROM_CHECKPOINT_ID = "fromcheckpointid"; public static final String FULL_PATH = "fullpath"; public static final String GATEWAY = "gateway"; public static final String IP6_GATEWAY = "ip6gateway"; @@ -335,6 +338,7 @@ public class ApiConstants { public static final String IS_2FA_VERIFIED = "is2faverified"; public static final String IS_2FA_MANDATED = "is2famandated"; + public static final String IS_ACTIVE = "isactive"; public static final String IS_ASYNC = "isasync"; public static final String IP_AVAILABLE = "ipavailable"; public static final String IP_LIMIT = "iplimit"; @@ -611,6 +615,7 @@ public class ApiConstants { public static final String TENANT_NAME = "tenantname"; public static final String TOTAL = "total"; public static final String TOTAL_SUBNETS = "totalsubnets"; + public static final String TO_CHECKPOINT_ID = "tocheckpointid"; public static final String TOTAL_QUOTA = "totalquota"; public static final String TYPE = "type"; public static final String TRUST_STORE = "truststore"; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java b/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java index 18c96c371591..1ee41ac86c22 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java @@ -21,8 +21,11 @@ import javax.servlet.http.HttpSession; +import org.apache.cloudstack.context.CallContext; + import com.cloud.domain.Domain; import com.cloud.exception.CloudAuthenticationException; +import com.cloud.user.Account; import com.cloud.user.UserAccount; public interface ApiServerService { @@ -52,4 +55,20 @@ public ResponseObject loginUser(HttpSession session, String username, String pas String getDomainId(Map params); boolean isPostRequestsAndTimestampsEnforced(); + + AsyncCmdResult processAsyncCmd(BaseAsyncCmd cmdObj, Map params, CallContext ctx, Long callerUserId, Account caller) throws Exception; + + class AsyncCmdResult { + public final Long objectId; + public final String objectUuid; + public final BaseAsyncCmd asyncCmd; + public final long jobId; + + public AsyncCmdResult(Long objectId, String objectUuid, BaseAsyncCmd asyncCmd, long jobId) { + this.objectId = objectId; + this.objectUuid = objectUuid; + this.asyncCmd = asyncCmd; + this.jobId = jobId; + } + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCustomIdCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCustomIdCmd.java index 30baa71d6d85..b9b13dcfd884 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCustomIdCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCustomIdCmd.java @@ -20,7 +20,7 @@ public abstract class BaseAsyncCreateCustomIdCmd extends BaseAsyncCreateCmd { @Parameter(name = ApiConstants.CUSTOM_ID, type = CommandType.STRING, description = "An optional field, in case you want to set a custom id to the resource. Allowed to Root Admins only") - private String customId; + protected String customId; public String getCustomId() { return customId; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java new file mode 100644 index 000000000000..c98bfb850529 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java @@ -0,0 +1,100 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ImageTransferResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.backup.ImageTransfer; +import org.apache.cloudstack.backup.KVMBackupExportService; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.utils.EnumUtils; + +@APICommand(name = "createImageTransfer", + description = "Create image transfer for a disk in backup. This API is intended for testing only and is disabled by default.", + responseObject = ImageTransferResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) +public class CreateImageTransferCmd extends BaseCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Parameter(name = ApiConstants.BACKUP_ID, + type = CommandType.UUID, + entityType = BackupResponse.class, + description = "ID of the backup") + private Long backupId; + + @Parameter(name = ApiConstants.VOLUME_ID, + type = CommandType.UUID, + entityType = VolumeResponse.class, + required = true, + description = "ID of the disk/volume") + private Long volumeId; + + @Parameter(name = ApiConstants.DIRECTION, + type = CommandType.STRING, + required = true, + description = "Direction of the transfer: upload, download") + private String direction; + + @Parameter(name = ApiConstants.FORMAT, + type = CommandType.STRING, + description = "Format for the image transfer: raw/cow. 'raw' will create an NBD backend. 'cow' will use the File backend." + + "For download, only the 'raw' format is supported. Default: raw") + private String format; + + public Long getBackupId() { + return backupId; + } + + public Long getVolumeId() { + return volumeId; + } + + public ImageTransfer.Direction getDirection() { + return ImageTransfer.Direction.valueOf(direction); + } + + public ImageTransfer.Format getFormat() { + return EnumUtils.getEnum(ImageTransfer.Format.class, format); + } + + @Override + public void execute() { + ImageTransferResponse response = kvmBackupExportService.createImageTransfer(this); + response.setObjectName(ImageTransfer.class.getSimpleName().toLowerCase()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java new file mode 100644 index 000000000000..d0e17e86d427 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java @@ -0,0 +1,85 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.KVMBackupExportService; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = "deleteVirtualMachineCheckpoint", + description = "Delete a VM checkpoint. This API is intended for testing only and is disabled by default.", + responseObject = SuccessResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) +public class DeleteVmCheckpointCmd extends BaseCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "ID of the VM") + private Long vmId; + + @Parameter(name = "checkpointid", + type = CommandType.STRING, + required = true, + description = "Checkpoint ID") + private String checkpointId; + + public Long getVmId() { + return vmId; + } + + public String getCheckpointId() { + return checkpointId; + } + + public void setVmId(Long vmId) { + this.vmId = vmId; + } + + public void setCheckpointId(String checkpointId) { + this.checkpointId = checkpointId; + } + + @Override + public void execute() { + boolean result = kvmBackupExportService.deleteVmCheckpoint(this); + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java new file mode 100644 index 000000000000..45173f8668ee --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java @@ -0,0 +1,103 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.backup.KVMBackupExportService; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.event.EventTypes; + +@APICommand(name = "finalizeBackup", + description = "Finalize a VM backup session. This API is intended for testing only and is disabled by default.", + responseObject = BackupResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) +public class FinalizeBackupCmd extends BaseAsyncCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Inject + private BackupManager backupManager; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "ID of the VM") + private Long vmId; + + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = BackupResponse.class, + required = true, + description = "ID of the backup") + private Long backupId; + + public Long getVmId() { + return vmId; + } + + public Long getBackupId() { + return backupId; + } + + @Override + public void execute() { + Backup backup = kvmBackupExportService.finalizeBackup(this); + + if (backup == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create Backup"); + } + + BackupResponse response = backupManager.createBackupResponse(backup, null); + + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + + @Override + public String getEventType() { + return EventTypes.EVENT_VM_BACKUP_CREATE; + } + + @Override + public String getEventDescription() { + return "Finalizing backup " + backupId; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java new file mode 100644 index 000000000000..dfc43e233bf2 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java @@ -0,0 +1,69 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.ImageTransferResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.backup.ImageTransfer; +import org.apache.cloudstack.backup.KVMBackupExportService; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = "finalizeImageTransfer", + description = "Finalize an image transfer. This API is intended for testing only and is disabled by default.", + responseObject = SuccessResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) +public class FinalizeImageTransferCmd extends BaseCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = ImageTransferResponse.class, + required = true, + description = "ID of the image transfer") + private Long imageTransferId; + + public Long getImageTransferId() { + return imageTransferId; + } + + @Override + public void execute() { + boolean result = kvmBackupExportService.finalizeImageTransfer(this); + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); + response.setObjectName(ImageTransfer.class.getSimpleName().toLowerCase()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java new file mode 100644 index 000000000000..d810d21ab5f8 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java @@ -0,0 +1,81 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ImageTransferResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.backup.ImageTransfer; +import org.apache.cloudstack.backup.KVMBackupExportService; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = "listImageTransfers", + description = "List image transfers for a backup. This API is intended for testing only and is disabled by default.", + responseObject = ImageTransferResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) +public class ListImageTransfersCmd extends BaseListCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = ImageTransferResponse.class, + description = "ID of the Image Transfer") + private Long id; + + @Parameter(name = ApiConstants.BACKUP_ID, + type = CommandType.UUID, + entityType = BackupResponse.class, + description = "ID of the backup") + private Long backupId; + + public Long getId() { + return id; + } + + public Long getBackupId() { + return backupId; + } + + @Override + public void execute() { + List responses = kvmBackupExportService.listImageTransfers(this); + ListResponse response = new ListResponse<>(); + response.setResponses(responses); + response.setObjectName(ImageTransfer.class.getSimpleName().toLowerCase()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java new file mode 100644 index 000000000000..a61661e982de --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java @@ -0,0 +1,69 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.CheckpointResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.KVMBackupExportService; + +@APICommand(name = "listVirtualMachineCheckpoints", + description = "List checkpoints for a VM. This API is intended for testing only and is disabled by default.", + responseObject = CheckpointResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) +public class ListVmCheckpointsCmd extends BaseListCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "ID of the VM") + private Long vmId; + + public Long getVmId() { + return vmId; + } + + @Override + public void execute() { + List responses = kvmBackupExportService.listVmCheckpoints(this); + ListResponse response = new ListResponse<>(); + response.setResponses(responses); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return 0; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java new file mode 100644 index 000000000000..1bf6d45db049 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java @@ -0,0 +1,120 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCreateCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.backup.KVMBackupExportService; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.event.EventTypes; + +@APICommand(name = "startBackup", + description = "Start a VM backup session using pull mode backup-begin on the KVM host. This API is intended for testing only and is disabled by default.", + responseObject = BackupResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) + public class StartBackupCmd extends BaseAsyncCreateCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Inject + private BackupManager backupManager; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "ID of the VM") + private Long vmId; + + @Parameter(name = ApiConstants.NAME, + type = CommandType.STRING, + description = "the name of the backup") + private String name; + + @Parameter(name = ApiConstants.DESCRIPTION, + type = CommandType.STRING, + description = "the description for the backup") + private String description; + + public Long getVmId() { + return vmId; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + @Override + public void execute() { + try { + Backup backup = kvmBackupExportService.startBackup(this); + BackupResponse response = backupManager.createBackupResponse(backup, null); + + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public void create() { + Backup backup = kvmBackupExportService.createBackup(this); + + if (backup != null) { + setEntityId(backup.getId()); + setEntityUuid(backup.getUuid()); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create Backup"); + } + } + + @Override + public String getEventType() { + return EventTypes.EVENT_VM_BACKUP_CREATE; + } + + @Override + public String getEventDescription() { + return "Starting backup for Instance " + vmId; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java index f6f66415f533..055443934609 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java @@ -19,23 +19,23 @@ import java.util.ArrayList; import java.util.List; -import org.apache.cloudstack.api.ApiErrorCode; -import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.response.DomainResponse; -import org.apache.commons.lang3.StringUtils; - import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseListCmd; import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.ConfigurationResponse; +import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ImageStoreResponse; import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.ManagementServerResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.config.Configuration; +import org.apache.commons.lang3.StringUtils; import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.Pair; @@ -94,6 +94,13 @@ public class ListCfgsByCmd extends BaseListCmd { description = "The ID of the Image Store to update the parameter value for corresponding image store") private Long imageStoreId; + @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, + type = CommandType.UUID, + entityType = ManagementServerResponse.class, + description = "the ID of the Management Server to update the parameter value for corresponding management server", + since = "4.23.0") + private Long managementServerId; + @Parameter(name = ApiConstants.GROUP, type = CommandType.STRING, description = "Lists configuration by group name (primarily used for UI)", since = "4.18.0") private String groupName; @@ -139,6 +146,10 @@ public Long getImageStoreId() { return imageStoreId; } + public Long getManagementServerId() { + return managementServerId; + } + public String getGroupName() { return groupName; } @@ -200,6 +211,9 @@ private void setScope(ConfigurationResponse cfgResponse) { if (getImageStoreId() != null){ cfgResponse.setScope("imagestore"); } + if (getManagementServerId() != null){ + cfgResponse.setScope("managementserver"); + } } @Override diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java index 2d511cff34db..3c3c36b29d7e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java @@ -23,16 +23,16 @@ import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.response.ImageStoreResponse; -import org.apache.cloudstack.framework.config.ConfigKey; - import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.ConfigurationResponse; import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.ManagementServerResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.config.Configuration; +import org.apache.cloudstack.framework.config.ConfigKey; import com.cloud.user.Account; import com.cloud.utils.Pair; @@ -84,6 +84,13 @@ public class ResetCfgCmd extends BaseCmd { description = "The ID of the Image Store to reset the parameter value for corresponding image store") private Long imageStoreId; + @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, + type = CommandType.UUID, + entityType = ManagementServerResponse.class, + description = "the ID of the Management Server to update the parameter value for corresponding management server", + since = "4.23.0") + private Long managementServerId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -116,6 +123,10 @@ public Long getImageStoreId() { return imageStoreId; } + public Long getManagementServerId() { + return managementServerId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -149,6 +160,9 @@ public void execute() { if (getImageStoreId() != null) { response.setScope(ConfigKey.Scope.ImageStore.name()); } + if (getManagementServerId() != null) { + response.setScope(ConfigKey.Scope.ManagementServer.name()); + } response.setValue(cfg.second()); this.setResponseObject(response); } else { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java index 97dee8f638af..9db9529dc8d8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java @@ -16,9 +16,7 @@ // under the License. package org.apache.cloudstack.api.command.admin.config; -import com.cloud.utils.crypt.DBEncryptionUtil; import org.apache.cloudstack.acl.RoleService; -import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiArgValidator; import org.apache.cloudstack.api.ApiConstants; @@ -29,13 +27,17 @@ import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.ConfigurationResponse; +import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.ManagementServerResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.config.Configuration; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.commons.lang3.StringUtils; import com.cloud.user.Account; +import com.cloud.utils.crypt.DBEncryptionUtil; @APICommand(name = "updateConfiguration", description = "Updates a configuration.", responseObject = ConfigurationResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -88,6 +90,13 @@ public class UpdateCfgCmd extends BaseCmd { validations = ApiArgValidator.PositiveNumber) private Long imageStoreId; + @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, + type = CommandType.UUID, + entityType = ManagementServerResponse.class, + description = "the ID of the Management Server to update the parameter value for corresponding management server", + since = "4.23.0") + private Long managementServerId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -112,7 +121,7 @@ public Long getClusterId() { return clusterId; } - public Long getStoragepoolId() { + public Long getStoragePoolId() { return storagePoolId; } @@ -128,6 +137,10 @@ public Long getImageStoreId() { return imageStoreId; } + public Long getManagementServerId() { + return managementServerId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -182,7 +195,7 @@ public ConfigurationResponse setResponseScopes(ConfigurationResponse response) { if (getClusterId() != null) { response.setScope("cluster"); } - if (getStoragepoolId() != null) { + if (getStoragePoolId() != null) { response.setScope("storagepool"); } if (getAccountId() != null) { @@ -191,6 +204,9 @@ public ConfigurationResponse setResponseScopes(ConfigurationResponse response) { if (getDomainId() != null) { response.setScope("domain"); } + if (getManagementServerId() != null) { + response.setScope(ConfigKey.Scope.ManagementServer.name().toLowerCase()); + } return response; } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java index e11d20d06466..50ff97ac676f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java @@ -85,6 +85,9 @@ public class AssignVMCmd extends BaseCmd { "In case no security groups are provided the Instance is part of the default security group.") private List securityGroupIdList; + // Internal flag to allow assignment without adding a network + private boolean skipNetwork = false; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -113,6 +116,34 @@ public List getSecurityGroupIdList() { return securityGroupIdList; } + public boolean isSkipNetwork() { + return skipNetwork; + } + + ///////////////////////////////////////////////////// + /////////////////// Setters ///////////////////////// + ///////////////////////////////////////////////////// + + public void setVirtualMachineId(Long virtualMachineId) { + this.virtualMachineId = virtualMachineId; + } + + public void setAccountName(String accountName) { + this.accountName = accountName; + } + + public void setDomainId(Long domainId) { + this.domainId = domainId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public void setSkipNetwork(boolean skipNetwork) { + this.skipNetwork = skipNetwork; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdmin.java index e64c8b3f46c6..b7c3b2ba41a6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdmin.java @@ -41,6 +41,19 @@ public class DeployVMCmdByAdmin extends DeployVMCmd implements AdminCmd { @Parameter(name = ApiConstants.CLUSTER_ID, type = CommandType.UUID, entityType = ClusterResponse.class, description = "Destination Cluster ID to deploy the Instance to - parameter available for root admin only", since = "4.13") private Long clusterId; + @Parameter(name = ApiConstants.BLANK_INSTANCE, + type = CommandType.BOOLEAN, + description = "Whether to create a blank instance without storage and network", + since = "4.23.0") + private Boolean blankInstance; + + // Internal flag to allow deploying instance with a given type + private String instanceType; + + ///////////////////////////////////////////////////// + ////////////////// Getters ////////////////////////// + ///////////////////////////////////////////////////// + public Long getPodId() { return podId; } @@ -48,4 +61,37 @@ public Long getPodId() { public Long getClusterId() { return clusterId; } + + @Override + public boolean isBlankInstance() { + return Boolean.TRUE.equals(blankInstance); + } + + @Override + public String getInstanceType() { + if (!isBlankInstance()) { + return null; + } + return instanceType; + } + + ///////////////////////////////////////////////////// + ////////////////// Setters ////////////////////////// + ///////////////////////////////////////////////////// + + public void setClusterId(Long clusterId) { + this.clusterId = clusterId; + } + + public void setBlankInstance(boolean blankInstance) { + this.blankInstance = blankInstance; + } + + public void setInstanceType(String instanceType) { + this.instanceType = instanceType; + } + + public void setCustomId(String customId) { + this.customId = customId; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java index 93d4b610b900..cbe6494d4004 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.api.command.admin.vm; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; @@ -27,4 +29,20 @@ @APICommand(name = "destroyVirtualMachine", description = "Destroys an Instance. Once destroyed, only the administrator can recover it.", responseObject = UserVmResponse.class, responseView = ResponseView.Full, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class DestroyVMCmdByAdmin extends DestroyVMCmd implements AdminCmd {} +public class DestroyVMCmdByAdmin extends DestroyVMCmd implements AdminCmd { + + @Parameter( name = ApiConstants.FORCED, + type = CommandType.BOOLEAN, + description = "Force destroy the Instance", + since = "4.23.0") + Boolean forced; + + @Override + public boolean isForced() { + return Boolean.TRUE.equals(forced); + } + + public void setForced(Boolean forced) { + this.forced = forced; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java index 5c5c8776bce3..cd6baaf827bb 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java @@ -193,6 +193,22 @@ public Boolean getGpuEnabled() { return gpuEnabled; } + public void setZoneId(Long zoneId) { + this.zoneId = zoneId; + } + + public void setCpuNumber(Integer cpuNumber) { + this.cpuNumber = cpuNumber; + } + + public void setMemory(Integer memory) { + this.memory = memory; + } + + public void setEncryptRoot(Boolean encryptRoot) { + this.encryptRoot = encryptRoot; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/AddNicToVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/AddNicToVMCmd.java index 6347c38811e8..f6ef955956f6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/AddNicToVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/AddNicToVMCmd.java @@ -100,6 +100,26 @@ public String getMacAddress() { return NetUtils.standardizeMacAddress(macaddr); } + public void setVmId(Long vmId) { + this.vmId = vmId; + } + + public void setNetworkId(Long netId) { + this.netId = netId; + } + + public void setIpaddr(String ipaddr) { + this.ipaddr = ipaddr; + } + + public void setMacAddress(String macaddr) { + this.macaddr = macaddr; + } + + public void setDhcpOptions(Map dhcpOptions) { + this.dhcpOptions = dhcpOptions; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java index 8c29d7338b85..68b0821ba44d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java @@ -61,10 +61,10 @@ import com.cloud.network.Network.IpAddresses; import com.cloud.offering.DiskOffering; import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.net.Dhcp; import com.cloud.utils.net.NetUtils; import com.cloud.vm.VmDetailConstants; import com.cloud.vm.VmDiskInfo; -import com.cloud.utils.net.Dhcp; public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityGroupAction, UserCmd { @@ -75,13 +75,13 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme ///////////////////////////////////////////////////// @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, required = true, description = "availability zone for the virtual machine") - private Long zoneId; + protected Long zoneId; @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "host name for the virtual machine", validations = {ApiArgValidator.RFCComplianceDomainName}) - private String name; + protected String name; @Parameter(name = ApiConstants.DISPLAY_NAME, type = CommandType.STRING, description = "an optional user generated name for the virtual machine") - private String displayName; + protected String displayName; @Parameter(name=ApiConstants.PASSWORD, type=CommandType.STRING, description="The password of the virtual machine. If null, a random password will be generated for the VM.", since="4.19.0.0") @@ -89,21 +89,21 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme //Owner information @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the virtual machine. Must be used with domainId.") - private String accountName; + protected String accountName; @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "an optional domainId for the virtual machine. If the account parameter is used, domainId must also be used. If account is NOT provided then virtual machine will be assigned to the caller account and domain.") - private Long domainId; + protected Long domainId; //Network information //@ACL(accessType = AccessType.UseEntry) @Parameter(name = ApiConstants.NETWORK_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = NetworkResponse.class, description = "list of network ids used by virtual machine. Can't be specified with ipToNetworkList parameter") - private List networkIds; + protected List networkIds; @Parameter(name = ApiConstants.BOOT_TYPE, type = CommandType.STRING, required = false, description = "Guest VM Boot option either custom[UEFI] or default boot [BIOS]. Not applicable with VMware if the template is marked as deploy-as-is, as we honour what is defined in the template.", since = "4.14.0.0") - private String bootType; + protected String bootType; @Parameter(name = ApiConstants.BOOT_MODE, type = CommandType.STRING, required = false, description = "Boot Mode [Legacy] or [Secure] Applicable when Boot Type Selected is UEFI, otherwise Legacy only for BIOS. Not applicable with VMware if the template is marked as deploy-as-is, as we honour what is defined in the template.", since = "4.14.0.0") - private String bootMode; + protected String bootMode; @Parameter(name = ApiConstants.BOOT_INTO_SETUP, type = CommandType.BOOLEAN, required = false, description = "Boot into hardware setup or not (ignored if startVm = false, only valid for vmware)", since = "4.15.0.0") private Boolean bootIntoSetup; @@ -138,7 +138,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme @Parameter(name = ApiConstants.HYPERVISOR, type = CommandType.STRING, description = "the hypervisor on which to deploy the virtual machine. " + "The parameter is required and respected only when hypervisor info is not set on the ISO/Template passed to the call") - private String hypervisor; + protected String hypervisor; @Parameter(name = ApiConstants.USER_DATA, type = CommandType.STRING, description = "an optional binary data that can be sent to the virtual machine upon a successful deployment. " + @@ -147,10 +147,10 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " + "You also need to change vm.userdata.max.length value", length = 1048576) - private String userData; + protected String userData; @Parameter(name = ApiConstants.USER_DATA_ID, type = CommandType.UUID, entityType = UserDataResponse.class, description = "the ID of the Userdata", since = "4.18") - private Long userdataId; + protected Long userdataId; @Parameter(name = ApiConstants.USER_DATA_DETAILS, type = CommandType.MAP, description = "used to specify the parameters values for the variables in userdata.", since = "4.18") private Map userdataDetails; @@ -160,7 +160,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme private String sshKeyPairName; @Parameter(name = ApiConstants.SSH_KEYPAIRS, type = CommandType.LIST, collectionType = CommandType.STRING, since="4.17", description = "names of the ssh key pairs used to login to the virtual machine") - private List sshKeyPairNames; + protected List sshKeyPairNames; @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, description = "destination Host ID to deploy the VM to - parameter available for root admin only") private Long hostId; @@ -168,7 +168,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme @ACL @Parameter(name = ApiConstants.SECURITY_GROUP_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = SecurityGroupResponse.class, description = "comma separated list of security groups id that going to be applied to the virtual machine. " + "Should be passed only when vm is created from a zone with Basic Network support." + " Mutually exclusive with securitygroupnames parameter") - private List securityGroupIdList; + protected List securityGroupIdList; @ACL @Parameter(name = ApiConstants.SECURITY_GROUP_NAMES, type = CommandType.LIST, collectionType = CommandType.STRING, entityType = SecurityGroupResponse.class, description = "comma separated list of security groups names that going to be applied to the virtual machine." @@ -189,10 +189,10 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme private String macAddress; @Parameter(name = ApiConstants.KEYBOARD, type = CommandType.STRING, description = "an optional keyboard device type for the virtual machine. valid value can be one of de,de-ch,es,es-latam,fi,fr,fr-be,fr-ch,is,it,jp,nl-be,no,pt,uk,us") - private String keyboard; + protected String keyboard; @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "Deploy vm for the project") - private Long projectId; + protected Long projectId; @Parameter(name = ApiConstants.START_VM, type = CommandType.BOOLEAN, description = "true if start vm after creating; defaulted to true if not specified") private Boolean startVm; @@ -200,7 +200,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme @ACL @Parameter(name = ApiConstants.AFFINITY_GROUP_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = AffinityGroupResponse.class, description = "comma separated list of affinity groups id that are going to be applied to the virtual machine." + " Mutually exclusive with affinitygroupnames parameter") - private List affinityGroupIdList; + protected List affinityGroupIdList; @ACL @Parameter(name = ApiConstants.AFFINITY_GROUP_NAMES, type = CommandType.LIST, collectionType = CommandType.STRING, entityType = AffinityGroupResponse.class, description = "comma separated list of affinity groups names that are going to be applied to the virtual machine." @@ -208,10 +208,10 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme private List affinityGroupNameList; @Parameter(name = ApiConstants.DISPLAY_VM, type = CommandType.BOOLEAN, since = "4.2", description = "an optional field, whether to the display the vm to the end user or not.", authorized = {RoleType.Admin}) - private Boolean displayVm; + protected Boolean displayVm; @Parameter(name = ApiConstants.DETAILS, type = CommandType.MAP, since = "4.3", description = "used to specify the custom parameters. 'extraconfig' is not allowed to be passed in details") - private Map details; + protected Map details; @Parameter(name = ApiConstants.DEPLOYMENT_PLANNER, type = CommandType.STRING, description = "Deployment planner to use for vm allocation. Available to ROOT admin only", since = "4.4", authorized = { RoleType.Admin }) private String deploymentPlanner; @@ -225,7 +225,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme private Map dataDiskTemplateToDiskOfferingList; @Parameter(name = ApiConstants.EXTRA_CONFIG, type = CommandType.STRING, since = "4.12", description = "an optional URL encoded string that can be passed to the virtual machine upon successful deployment", length = 5120) - private String extraConfig; + protected String extraConfig; @Parameter(name = ApiConstants.COPY_IMAGE_TAGS, type = CommandType.BOOLEAN, since = "4.13", description = "if true the image tags (if any) will be copied to the VM, default value is false") private Boolean copyImageTags; @@ -798,6 +798,11 @@ public IoDriverPolicy getIoDriverPolicy() { } return null; } + + public String getInstanceType() { + return null; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java index 050592b97a3b..0a943fab118d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java @@ -16,10 +16,11 @@ // under the License. package org.apache.cloudstack.api.command.user.vm; +import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.stream.Stream; -import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; @@ -40,6 +41,7 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.uservm.UserVm; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; @APICommand(name = "deployVirtualMachine", description = "Creates and automatically starts an Instance based on a service offering, disk offering, and Template.", responseObject = UserVmResponse.class, responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, @@ -88,6 +90,111 @@ public boolean isVolumeOrSnapshotProvided() { return volumeId != null || snapshotId != null; } + public boolean isBlankInstance() { + return false; + } + + + + ///////////////////////////////////////////////////// + ////////////////// Setters ////////////////////////// + ///////////////////////////////////////////////////// + public void setZoneId(Long zoneId) { + this.zoneId = zoneId; + } + + public void setName(String name) { + this.name = name; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + public void setAccountName(String accountName) { + this.accountName = accountName; + } + + public void setDomainId(Long domainId) { + this.domainId = domainId; + } + + public void setNetworkIds(List networkIds) { + this.networkIds = networkIds; + } + + public void setBootType(String bootType) { + this.bootType = bootType; + } + + public void setBootMode(String bootMode) { + this.bootMode = bootMode; + } + + public void setHypervisor(String hypervisor) { + this.hypervisor = hypervisor; + } + + public void setUserData(String userData) { + this.userData = userData; + } + + public void setKeyboard(String keyboard) { + this.keyboard = keyboard; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public void setDisplayVm(Boolean displayVm) { + this.displayVm = displayVm; + } + + public void setUserDataId(Long userDataId) { + this.userdataId = userDataId; + } + + public void setAffinityGroupIds(List ids) { + this.affinityGroupIdList = ids; + } + + public void setDetails(Map details) { + this.details = details; + } + + public void setExtraConfig(String extraConfig) { + this.extraConfig = extraConfig; + } + + public void setDynamicScalingEnabled(Boolean dynamicScalingEnabled) { + this.dynamicScalingEnabled = dynamicScalingEnabled; + } + + public void setServiceOfferingId(Long serviceOfferingId) { + this.serviceOfferingId = serviceOfferingId; + } + + public void setTemplateId(Long templateId) { + this.templateId = templateId; + } + + public void setVolumeId(Long volumeId) { + this.volumeId = volumeId; + } + + public void setSnapshotId(Long snapshotId) { + this.snapshotId = snapshotId; + } + + public void setSshKeyPairNames(List sshKeyPairNames) { + this.sshKeyPairNames = sshKeyPairNames; + } + + public void setSecurityGroupList(List securityGroupIdList) { + this.securityGroupIdList = securityGroupIdList; + } + @Override public void execute() { UserVm result; @@ -132,7 +239,7 @@ public void execute() { @Override public void create() throws ResourceAllocationException { - if (Stream.of(templateId, snapshotId, volumeId).filter(Objects::nonNull).count() != 1) { + if (!isBlankInstance() && Stream.of(templateId, snapshotId, volumeId).filter(Objects::nonNull).count() != 1) { throw new CloudRuntimeException("Please provide only one of the following parameters - template ID, volume ID or snapshot ID"); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java index 9e2f2bcb72ce..aec0688f1779 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java @@ -90,6 +90,10 @@ public List getVolumeIds() { return volumeIds; } + public boolean isForced() { + return false; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java index e3ad0502f454..a5f06e6ecbb3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java @@ -288,6 +288,14 @@ public boolean isCleanupExtraConfig() { return Boolean.TRUE.equals(cleanupExtraConfig); } + public void setId(Long id) { + this.id = id; + } + + public void setSecurityGroupIdList(List securityGroupIdList) { + this.securityGroupIdList = securityGroupIdList; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AssignVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AssignVolumeCmd.java index f39853512281..f50abaf73c96 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AssignVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AssignVolumeCmd.java @@ -70,6 +70,21 @@ public Long getProjectid() { return projectid; } + ///////////////////////////////////////////////////// + /////////////////// Setter/////////////////////////// + ///////////////////////////////////////////////////// + public void setVolumeId(Long volumeId) { + this.volumeId = volumeId; + } + + public void setAccountId(Long accountId) { + this.accountId = accountId; + } + + public void setProjectId(Long projectid) { + this.projectid = projectid; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java index 78de05648486..ec7a626fa156 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java @@ -114,7 +114,8 @@ public class CreateVolumeCmd extends BaseAsyncCreateCustomIdCmd implements UserC type = CommandType.UUID, entityType = StoragePoolResponse.class, description = "Storage pool ID to create the volume in. Cannot be used with the snapshotid parameter.", - authorized = {RoleType.Admin}) + authorized = {RoleType.Admin}, + since = "4.22.1") private Long storageId; ///////////////////////////////////////////////////// @@ -150,6 +151,10 @@ public Long getMaxIops() { } public Long getSnapshotId() { + if (storageId != null && snapshotId != null) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Snapshot ID cannot be specified with the Storage ID."); + } return snapshotId; } @@ -163,7 +168,8 @@ private Long getProjectId() { public Long getStorageId() { if (snapshotId != null && storageId != null) { - throw new IllegalArgumentException("StorageId parameter cannot be specified with the SnapshotId parameter."); + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Storage ID cannot be specified with the Snapshot ID."); } return storageId; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DetachVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DetachVolumeCmd.java index 66a558abf982..f6e811da6055 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DetachVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DetachVolumeCmd.java @@ -77,6 +77,10 @@ public Long getVirtualMachineId() { return virtualMachineId; } + public void setId(Long id) { + this.id = id; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java index b855bfe40b8d..51fcaa9836e9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java @@ -127,6 +127,18 @@ public class BackupResponse extends BaseResponse { @Param(description = "Indicates whether the VM from which the backup was taken is expunged or not", since = "4.22.0") private Boolean isVmExpunged; + @SerializedName(ApiConstants.FROM_CHECKPOINT_ID) + @Param(description = "Previous active checkpoint ID for incremental backups", since = "4.23.0") + private String fromCheckpointId; + + @SerializedName(ApiConstants.TO_CHECKPOINT_ID) + @Param(description = "Next checkpoint ID for incremental backups", since = "4.23.0") + private String toCheckpointId; + + @SerializedName(ApiConstants.HOST_ID) + @Param(description = "Host ID where the backup is running", since = "4.23.0") + private String hostId; + public String getId() { return id; } @@ -314,4 +326,28 @@ public void setVmOfferingRemoved(Boolean vmOfferingRemoved) { public void setVmExpunged(Boolean isVmExpunged) { this.isVmExpunged = isVmExpunged; } + + public void setFromCheckpointId(String fromCheckpointId) { + this.fromCheckpointId = fromCheckpointId; + } + + public String getFromCheckpointId() { + return this.fromCheckpointId; + } + + public void setToCheckpointId(String toCheckpointId) { + this.toCheckpointId = toCheckpointId; + } + + public String getToCheckpointId() { + return this.toCheckpointId; + } + + public void setHostId(String hostId) { + this.hostId = hostId; + } + + public String getHostId() { + return this.hostId; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java new file mode 100644 index 000000000000..2bec7711064f --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java @@ -0,0 +1,53 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.response; + +import java.util.Date; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +public class CheckpointResponse extends BaseResponse { + + @SerializedName(ApiConstants.ID) + @Param(description = "the checkpoint ID") + private String id; + + @SerializedName(ApiConstants.CREATED) + @Param(description = "the checkpoint creation time") + private Date created; + + @SerializedName(ApiConstants.IS_ACTIVE) + @Param(description = "whether this is the active checkpoint") + private Boolean isActive; + + public void setId(String id) { + this.id = id; + } + + public void setCreated(Date created) { + this.created = created; + } + + public void setIsActive(Boolean isActive) { + this.isActive = isActive; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java new file mode 100644 index 000000000000..15576e8f1012 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java @@ -0,0 +1,104 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.response; + +import java.util.Date; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; +import org.apache.cloudstack.backup.ImageTransfer; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +@EntityReference(value = ImageTransfer.class) +public class ImageTransferResponse extends BaseResponse { + + @SerializedName(ApiConstants.ID) + @Param(description = "the ID of the image transfer") + private String id; + + @SerializedName("backupid") + @Param(description = "the backup ID") + private String backupId; + + @SerializedName("vmid") + @Param(description = "the VM ID") + private String vmId; + + @SerializedName(ApiConstants.VOLUME_ID) + @Param(description = "the disk/volume ID") + private String diskId; + + @SerializedName("devicename") + @Param(description = "the device name (vda, vdb, etc)") + private String deviceName; + + @SerializedName("transferurl") + @Param(description = "the transfer URL") + private String transferUrl; + + @SerializedName("phase") + @Param(description = "the transfer phase") + private String phase; + + @SerializedName("direction") + @Param(description = "the image transfer direction: upload / download") + private String direction; + + @SerializedName(ApiConstants.CREATED) + @Param(description = "the date created") + private Date created; + + public void setId(String id) { + this.id = id; + } + + public void setBackupId(String backupId) { + this.backupId = backupId; + } + + public void setVmId(String vmId) { + this.vmId = vmId; + } + + public void setDiskId(String diskId) { + this.diskId = diskId; + } + + public void setDeviceName(String deviceName) { + this.deviceName = deviceName; + } + + public void setTransferUrl(String transferUrl) { + this.transferUrl = transferUrl; + } + + public void setPhase(String phase) { + this.phase = phase; + } + + public void setDirection(String direction) { + this.direction = direction; + } + + public void setCreated(Date created) { + this.created = created; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/backup/Backup.java b/api/src/main/java/org/apache/cloudstack/backup/Backup.java index 951af9180e7f..2d68f18b953f 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/Backup.java +++ b/api/src/main/java/org/apache/cloudstack/backup/Backup.java @@ -30,8 +30,16 @@ public interface Backup extends ControlledEntity, InternalIdentity, Identity { + String getFromCheckpointId(); + + String getToCheckpointId(); + + Long getCheckpointCreateTime(); + + Long getHostId(); + enum Status { - Allocated, Queued, BackingUp, BackedUp, Error, Failed, Restoring, Removed, Expunged + Allocated, Queued, BackingUp, ReadyForImageTransfer, FinalizingImageTransfer, BackedUp, Error, Failed, Restoring, Removed, Expunged } class Metric { diff --git a/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java b/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java new file mode 100644 index 000000000000..e1153be3ae02 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.api.InternalIdentity; + +public interface ImageTransfer extends ControlledEntity, InternalIdentity { + long getDataCenterId(); + + public enum Direction { + upload, download + } + + public enum Format { + raw, + cow + } + + public enum Backend { + nbd, + file + } + + public enum Phase { + initializing, transferring, finished, failed + } + + String getUuid(); + + Long getBackupId(); + + long getVolumeId(); + + long getHostId(); + + String getTransferUrl(); + + Phase getPhase(); + + Direction getDirection(); + + Backend getBackend(); + + String getSignedTicketId(); +} diff --git a/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java new file mode 100644 index 000000000000..e1affb311ec6 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java @@ -0,0 +1,106 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.List; + +import org.apache.cloudstack.api.command.admin.backup.CreateImageTransferCmd; +import org.apache.cloudstack.api.command.admin.backup.DeleteVmCheckpointCmd; +import org.apache.cloudstack.api.command.admin.backup.FinalizeBackupCmd; +import org.apache.cloudstack.api.command.admin.backup.FinalizeImageTransferCmd; +import org.apache.cloudstack.api.command.admin.backup.ListImageTransfersCmd; +import org.apache.cloudstack.api.command.admin.backup.ListVmCheckpointsCmd; +import org.apache.cloudstack.api.command.admin.backup.StartBackupCmd; +import org.apache.cloudstack.api.response.CheckpointResponse; +import org.apache.cloudstack.api.response.ImageTransferResponse; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; + +import com.cloud.utils.component.PluggableService; + +/** + * Service for Creating Backups and ImageTransfer sessions which will be consumed by an external orchestrator. + */ +public interface KVMBackupExportService extends Configurable, PluggableService { + + ConfigKey ImageTransferIdleTimeoutSeconds = new ConfigKey<>("Advanced", Integer.class, + "image.transfer.idle.timeout.seconds", + "600", + "Seconds since last completed HTTP request to an image transfer before the image server unregisters it (idle timeout).", + true, ConfigKey.Scope.Zone); + + ConfigKey ExposeKVMBackupExportServiceApis = new ConfigKey<>("Advanced", Boolean.class, + "expose.kvm.backup.export.service.apis", + "false", + "Enable to expose APIs for testing the KVM Backup Export Service.", + false, ConfigKey.Scope.Global); + /** + * Creates a backup session for a VM + */ + Backup createBackup(StartBackupCmd cmd); + + /** + * Start a backup session for a VM + * Creates a new checkpoint and starts NBD server for pull-mode backup + */ + Backup startBackup(StartBackupCmd cmd); + + /** + * Finalize a backup session + * Stops NBD server, updates checkpoint tracking, deletes old checkpoints + */ + Backup finalizeBackup(FinalizeBackupCmd cmd); + + /** + * Create an image transfer object for a disk + * Registers NBD endpoint with ImageIO (stubbed for POC) + */ + ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd); + + ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTransfer.Direction direction, ImageTransfer.Format format); + + boolean cancelImageTransfer(long imageTransferId); + + /** + * Finalize an image transfer + * Marks transfer as complete (NBD is closed globally in finalize backup) + */ + boolean finalizeImageTransfer(FinalizeImageTransferCmd cmd); + + boolean finalizeImageTransfer(long imageTransferId); + + /** + * List image transfers for a backup + */ + List listImageTransfers(ListImageTransfersCmd cmd); + + /** + * List checkpoints for a VM + */ + List listVmCheckpoints(ListVmCheckpointsCmd cmd); + + /** + * Delete a VM checkpoint (no-op for normal flow, kept for API parity) + */ + boolean deleteVmCheckpoint(DeleteVmCheckpointCmd cmd); + + /** + * List Compatible Data Center Ids for the service + */ + List listCompatibleDataCenterIds(); +} diff --git a/api/src/main/java/org/apache/cloudstack/storage/sharedfs/SharedFSService.java b/api/src/main/java/org/apache/cloudstack/storage/sharedfs/SharedFSService.java index 21184de27a26..3e349cf0bd39 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/sharedfs/SharedFSService.java +++ b/api/src/main/java/org/apache/cloudstack/storage/sharedfs/SharedFSService.java @@ -23,6 +23,10 @@ import org.apache.cloudstack.api.command.user.storage.sharedfs.ChangeSharedFSServiceOfferingCmd; import org.apache.cloudstack.api.command.user.storage.sharedfs.CreateSharedFSCmd; import org.apache.cloudstack.api.command.user.storage.sharedfs.DestroySharedFSCmd; +import org.apache.cloudstack.api.command.user.storage.sharedfs.ListSharedFSCmd; +import org.apache.cloudstack.api.command.user.storage.sharedfs.UpdateSharedFSCmd; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.SharedFSResponse; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.ManagementServerException; @@ -31,11 +35,6 @@ import com.cloud.exception.ResourceUnavailableException; import com.cloud.exception.VirtualMachineMigrationException; -import org.apache.cloudstack.api.command.user.storage.sharedfs.ListSharedFSCmd; -import org.apache.cloudstack.api.command.user.storage.sharedfs.UpdateSharedFSCmd; -import org.apache.cloudstack.api.response.SharedFSResponse; -import org.apache.cloudstack.api.response.ListResponse; - public interface SharedFSService { List getSharedFSProviders(); @@ -69,4 +68,10 @@ public interface SharedFSService { SharedFS recoverSharedFS(Long sharedFSId); void deleteSharedFS(Long sharedFSId); + + SharedFS getSharedFSByUuid(String uuid); + + SharedFS getSharedFSForVmId(long vmId); + + SharedFS updateSharedFSPostRestore(long sharedFsId, long volumeId); } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmdTest.java new file mode 100644 index 000000000000..27bc4614e1b5 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmdTest.java @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.vm; + +import org.junit.Assert; +import org.junit.Test; +import org.springframework.test.util.ReflectionTestUtils; + +public class AssignVMCmdTest { + + @Test + public void test_setSkipNetwork_default() { + AssignVMCmd assignVMCmd = new AssignVMCmd(); + Object value = ReflectionTestUtils.getField(assignVMCmd, "skipNetwork"); + Assert.assertTrue(value instanceof Boolean); + Assert.assertFalse((Boolean) value); + } + + @Test + public void test_setSkipNetwork_set() { + AssignVMCmd assignVMCmd = new AssignVMCmd(); + assignVMCmd.setSkipNetwork(true); + Object value = ReflectionTestUtils.getField(assignVMCmd, "skipNetwork"); + Assert.assertTrue(value instanceof Boolean); + Assert.assertTrue((Boolean) value); + } + + @Test + public void test_isSkipNetwork_default() { + AssignVMCmd assignVMCmd = new AssignVMCmd(); + Assert.assertFalse(assignVMCmd.isSkipNetwork()); + } + + @Test + public void test_isSkipNetwork_set() { + AssignVMCmd assignVMCmd = new AssignVMCmd(); + ReflectionTestUtils.setField(assignVMCmd, "skipNetwork", true); + Assert.assertTrue(assignVMCmd.isSkipNetwork()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdminTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdminTest.java new file mode 100644 index 000000000000..d82dc9f766f9 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdminTest.java @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.vm; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +@RunWith(MockitoJUnitRunner.class) +public class DeployVMCmdByAdminTest { + + @InjectMocks + private DeployVMCmdByAdmin cmd; + + @Test + public void testIsBlankInstance_default() { + assertFalse(cmd.isBlankInstance()); + } + + @Test + public void testIsBlankInstance_true() { + ReflectionTestUtils.setField(cmd, "blankInstance", true); + assertTrue(cmd.isBlankInstance()); + } + + @Test + public void testIsBlankInstance_false() { + ReflectionTestUtils.setField(cmd, "blankInstance", false); + assertFalse(cmd.isBlankInstance()); + } + + @Test + public void testSetBlankInstance_default() { + Object obj = ReflectionTestUtils.getField(cmd, "blankInstance"); + assertNull(obj); + } + + @Test + public void testSetBlankInstance_true() { + cmd.setBlankInstance(true); + Object obj = ReflectionTestUtils.getField(cmd, "blankInstance"); + assertNotNull(obj); + assertTrue((boolean)obj); + } + + @Test + public void testSetBlankInstance_false() { + cmd.setBlankInstance(false); + Object obj = ReflectionTestUtils.getField(cmd, "blankInstance"); + assertNotNull(obj); + assertFalse((boolean)obj); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmdTest.java index f7e3e38d9c3f..09d396e40237 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmdTest.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.command.user.vm; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; @@ -41,6 +42,7 @@ import org.springframework.test.util.ReflectionTestUtils; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.network.NetworkService; import com.cloud.utils.db.EntityManager; import com.cloud.vm.VmDetailConstants; @@ -480,4 +482,146 @@ public void testGetDataDiskTemplateToDiskOfferingMapInvalidTemplateId() { }); assertTrue(thrownException.getMessage().contains("Unable to translate and find entity with datadisktemplateid")); } + + @Test + public void testSetServiceOfferingId() { + cmd.setServiceOfferingId(101L); + assertEquals(Long.valueOf(101L), cmd.getServiceOfferingId()); + } + + @Test + public void testSetTemplateId() { + cmd.setTemplateId(102L); + assertEquals(Long.valueOf(102L), cmd.getTemplateId()); + } + + @Test + public void testSetVolumeId() { + cmd.setVolumeId(103L); + assertEquals(Long.valueOf(103L), cmd.getVolumeId()); + } + + @Test + public void testSetSnapshotId() { + cmd.setSnapshotId(104L); + assertEquals(Long.valueOf(104L), cmd.getSnapshotId()); + } + + @Test + public void testSetZoneId() { + cmd.setZoneId(105L); + assertEquals(Long.valueOf(105L), cmd.getZoneId()); + } + + @Test + public void testSetName() { + cmd.setName("vm-name"); + assertEquals("vm-name", cmd.getName()); + } + + @Test + public void testSetDisplayName() { + cmd.setDisplayName("vm-display-name"); + assertEquals("vm-display-name", cmd.getDisplayName()); + } + + @Test + public void testSetAccountName() { + cmd.setAccountName("account-name"); + assertEquals("account-name", cmd.getAccountName()); + } + + @Test + public void testSetDomainId() { + cmd.setDomainId(106L); + assertEquals(Long.valueOf(106L), cmd.getDomainId()); + } + + @Test + public void testSetNetworkIds() { + List networkIds = Arrays.asList(11L, 12L); + cmd.setNetworkIds(networkIds); + assertEquals(networkIds, cmd.getNetworkIds()); + } + + @Test + public void testSetBootType() { + cmd.setBootType("UEFI"); + assertEquals(BootType.UEFI, cmd.getBootType()); + } + + @Test + public void testSetBootMode() { + cmd.setBootType("UEFI"); + cmd.setBootMode("SECURE"); + assertEquals(BootMode.SECURE, cmd.getBootMode()); + } + + @Test + public void testSetHypervisor() { + cmd.setHypervisor("KVM"); + assertEquals(HypervisorType.KVM, cmd.getHypervisor()); + } + + @Test + public void testSetUserData() { + cmd.setUserData("dXNlci1kYXRh"); + assertEquals("dXNlci1kYXRh", cmd.getUserData()); + } + + @Test + public void testSetKeyboard() { + cmd.setKeyboard("us"); + assertEquals("us", cmd.getKeyboard()); + } + + @Test + public void testSetProjectId() { + cmd.setProjectId(107L); + assertEquals(Long.valueOf(107L), ReflectionTestUtils.getField(cmd, "projectId")); + } + + @Test + public void testSetDisplayVm() { + cmd.setDisplayVm(Boolean.FALSE); + assertEquals(Boolean.FALSE, cmd.isDisplayVm()); + } + + @Test + public void testSetUserDataId() { + cmd.setUserDataId(108L); + assertEquals(Long.valueOf(108L), cmd.getUserdataId()); + } + + @Test + public void testSetAffinityGroupIds() { + List affinityGroupIds = Arrays.asList(21L, 22L); + cmd.setAffinityGroupIds(affinityGroupIds); + assertEquals(affinityGroupIds, cmd.getAffinityGroupIdList()); + } + + @Test + public void testSetDetails() { + Map details = new HashMap<>(); + details.put("key", "value"); + cmd.setDetails(details); + assertEquals(details, ReflectionTestUtils.getField(cmd, "details")); + } + + @Test + public void testSetExtraConfig() { + cmd.setExtraConfig("cpu-mode=host-passthrough"); + assertEquals("cpu-mode=host-passthrough", cmd.getExtraConfig()); + } + + @Test + public void testSetDynamicScalingEnabled() { + cmd.setDynamicScalingEnabled(Boolean.FALSE); + assertFalse(cmd.isDynamicScalingEnabled()); + } + + @Test + public void testIsBlankInstance() { + assertFalse(cmd.isBlankInstance()); + } } diff --git a/client/pom.xml b/client/pom.xml index ba16cc5d34a2..7d37225dbbb3 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -612,6 +612,11 @@ cloud-plugin-backup-nas ${project.version} + + org.apache.cloudstack + cloud-plugin-integrations-veeam-control-service + ${project.version} + org.apache.cloudstack cloud-plugin-integrations-kubernetes-service diff --git a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferAnswer.java new file mode 100644 index 000000000000..34cf6d4ca34c --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferAnswer.java @@ -0,0 +1,56 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; + +public class CreateImageTransferAnswer extends Answer { + private String imageTransferId; + private String transferUrl; + + public CreateImageTransferAnswer() { + } + + public CreateImageTransferAnswer(CreateImageTransferCommand cmd, boolean success, String details) { + super(cmd, success, details); + } + + public CreateImageTransferAnswer(CreateImageTransferCommand cmd, boolean success, String details, + String imageTransferId, String transferUrl) { + super(cmd, success, details); + this.imageTransferId = imageTransferId; + this.transferUrl = transferUrl; + } + + public String getImageTransferId() { + return imageTransferId; + } + + public void setImageTransferId(String imageTransferId) { + this.imageTransferId = imageTransferId; + } + + public String getTransferUrl() { + return transferUrl; + } + + public void setTransferUrl(String transferUrl) { + this.transferUrl = transferUrl; + } + +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java new file mode 100644 index 000000000000..95b56c9a9c38 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java @@ -0,0 +1,94 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; + +public class CreateImageTransferCommand extends Command { + private String transferId; + private String exportName; + private String socket; + private String direction; + private String checkpointId; + private String file; + private ImageTransfer.Backend backend; + private int idleTimeoutSeconds; + + public CreateImageTransferCommand() { + } + + private CreateImageTransferCommand(String transferId, String direction, String socket, int idleTimeoutSeconds) { + this.transferId = transferId; + this.direction = direction; + this.socket = socket; + this.idleTimeoutSeconds = idleTimeoutSeconds; + } + + public CreateImageTransferCommand(String transferId, String direction, String exportName, String socket, String checkpointId, int idleTimeoutSeconds) { + this(transferId, direction, socket, idleTimeoutSeconds); + this.backend = ImageTransfer.Backend.nbd; + this.exportName = exportName; + this.checkpointId = checkpointId; + } + + public CreateImageTransferCommand(String transferId, String direction, String socket, String file, int idleTimeoutSeconds) { + this(transferId, direction, socket, idleTimeoutSeconds); + if (direction == ImageTransfer.Direction.download.toString()) { + throw new IllegalArgumentException("File backend is only supported for upload"); + } + this.backend = ImageTransfer.Backend.file; + this.file = file; + } + + public String getExportName() { + return exportName; + } + + public String getSocket() { + return socket; + } + + public String getFile() { + return file; + } + + public ImageTransfer.Backend getBackend() { + return backend; + } + + public String getTransferId() { + return transferId; + } + + @Override + public boolean executeInSequence() { + return true; + } + + public String getDirection() { + return direction; + } + + public String getCheckpointId() { + return checkpointId; + } + + public int getIdleTimeoutSeconds() { + return idleTimeoutSeconds; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/DeleteVmCheckpointCommand.java b/core/src/main/java/org/apache/cloudstack/backup/DeleteVmCheckpointCommand.java new file mode 100644 index 000000000000..81cf6c1abfcc --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/DeleteVmCheckpointCommand.java @@ -0,0 +1,60 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.Map; + +import com.cloud.agent.api.Command; + +public class DeleteVmCheckpointCommand extends Command { + private String vmName; + private String checkpointId; + private Map diskPathUuidMap; + private boolean stoppedVM; + + public DeleteVmCheckpointCommand() { + } + + public DeleteVmCheckpointCommand(String vmName, String checkpointId, Map diskPathUuidMap, boolean stoppedVM) { + this.vmName = vmName; + this.checkpointId = checkpointId; + this.diskPathUuidMap = diskPathUuidMap; + this.stoppedVM = stoppedVM; + } + + public String getVmName() { + return vmName; + } + + public String getCheckpointId() { + return checkpointId; + } + + public Map getDiskPathUuidMap() { + return diskPathUuidMap; + } + + public boolean isStoppedVM() { + return stoppedVM; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java new file mode 100644 index 000000000000..84d9b1ff8186 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java @@ -0,0 +1,40 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; + +public class FinalizeImageTransferCommand extends Command { + private String transferId; + + public FinalizeImageTransferCommand() { + } + + public FinalizeImageTransferCommand(String transferId) { + this.transferId = transferId; + } + + public String getTransferId() { + return transferId; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java new file mode 100644 index 000000000000..d7cbf097df90 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java @@ -0,0 +1,44 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; + +public class StartBackupAnswer extends Answer { + private Long checkpointCreateTime; + + public StartBackupAnswer() { + } + + public StartBackupAnswer(StartBackupCommand cmd, boolean success, String details) { + super(cmd, success, details); + } + + public StartBackupAnswer(StartBackupCommand cmd, boolean success, String details, Long checkpointCreateTime) { + super(cmd, success, details); + this.checkpointCreateTime = checkpointCreateTime; + } + + public Long getCheckpointCreateTime() { + return checkpointCreateTime; + } + + public void setCheckpointCreateTime(Long checkpointCreateTime) { + this.checkpointCreateTime = checkpointCreateTime; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java new file mode 100644 index 000000000000..6f1ed6834500 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java @@ -0,0 +1,91 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.Map; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.LogLevel; + +public class StartBackupCommand extends Command { + private String vmName; + private String toCheckpointId; + private String fromCheckpointId; + private Long fromCheckpointCreateTime; + private String socket; + private Map diskPathUuidMap; + private boolean stoppedVM; + @LogLevel(LogLevel.Log4jLevel.Off) + private Map diskPathPassphraseMap; + + public StartBackupCommand() { + } + + public StartBackupCommand(String vmName, String toCheckpointId, String fromCheckpointId, Long fromCheckpointCreateTime, + String socket, Map diskPathUuidMap, Map passphrases, boolean stoppedVM) { + this.vmName = vmName; + this.toCheckpointId = toCheckpointId; + this.fromCheckpointId = fromCheckpointId; + this.fromCheckpointCreateTime = fromCheckpointCreateTime; + this.socket = socket; + this.diskPathUuidMap = diskPathUuidMap; + this.diskPathPassphraseMap = passphrases; + this.stoppedVM = stoppedVM; + } + + public String getVmName() { + return vmName; + } + + public String getToCheckpointId() { + return toCheckpointId; + } + + public String getFromCheckpointId() { + return fromCheckpointId; + } + + public Long getFromCheckpointCreateTime() { + return fromCheckpointCreateTime; + } + + public String getSocket() { + return socket; + } + + public Map getDiskPathUuidMap() { + return diskPathUuidMap; + } + + public boolean isIncremental() { + return fromCheckpointId != null && !fromCheckpointId.isEmpty(); + } + + public boolean isStoppedVM() { + return stoppedVM; + } + + public Map getDiskPathPassphraseMap() { + return diskPathPassphraseMap; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerAnswer.java new file mode 100644 index 000000000000..d8c78d3c8807 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerAnswer.java @@ -0,0 +1,56 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; + +public class StartNBDServerAnswer extends Answer { + private String imageTransferId; + private String transferUrl; + + public StartNBDServerAnswer() { + } + + public StartNBDServerAnswer(StartNBDServerCommand cmd, boolean success, String details) { + super(cmd, success, details); + } + + public StartNBDServerAnswer(StartNBDServerCommand cmd, boolean success, String details, + String imageTransferId, String transferUrl) { + super(cmd, success, details); + this.imageTransferId = imageTransferId; + this.transferUrl = transferUrl; + } + + public String getImageTransferId() { + return imageTransferId; + } + + public void setImageTransferId(String imageTransferId) { + this.imageTransferId = imageTransferId; + } + + public String getTransferUrl() { + return transferUrl; + } + + public void setTransferUrl(String transferUrl) { + this.transferUrl = transferUrl; + } + +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java new file mode 100644 index 000000000000..07e50f11fd37 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java @@ -0,0 +1,78 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.LogLevel; + +public class StartNBDServerCommand extends Command { + private String transferId; + private String exportName; + private String volumePath; + private String socket; + private String direction; + private String fromCheckpointId; + @LogLevel(LogLevel.Log4jLevel.Off) + private byte[] passphrase; + + public StartNBDServerCommand() { + } + + protected StartNBDServerCommand(String transferId, String exportName, String volumePath, String socket, String direction, String fromCheckpointId, byte[] passphrase) { + this.transferId = transferId; + this.socket = socket; + this.exportName = exportName; + this.volumePath = volumePath; + this.direction = direction; + this.fromCheckpointId = fromCheckpointId; + this.passphrase = passphrase; + } + + public String getExportName() { + return exportName; + } + + public String getSocket() { + return socket; + } + + public String getTransferId() { + return transferId; + } + + @Override + public boolean executeInSequence() { + return true; + } + + public String getVolumePath() { + return volumePath; + } + + public String getDirection() { + return direction; + } + + public String getFromCheckpointId() { + return fromCheckpointId; + } + + public byte[] getPassphrase() { + return passphrase; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StopBackupAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/StopBackupAnswer.java new file mode 100644 index 000000000000..ce977f31e005 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StopBackupAnswer.java @@ -0,0 +1,30 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; + +public class StopBackupAnswer extends Answer { + + public StopBackupAnswer() { + } + + public StopBackupAnswer(StopBackupCommand cmd, boolean success, String details) { + super(cmd, success, details); + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StopBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StopBackupCommand.java new file mode 100644 index 000000000000..d3055021e9de --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StopBackupCommand.java @@ -0,0 +1,52 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; + +public class StopBackupCommand extends Command { + private String vmName; + private Long vmId; + private Long backupId; + + public StopBackupCommand() { + } + + public StopBackupCommand(String vmName, Long vmId, Long backupId) { + this.vmName = vmName; + this.vmId = vmId; + this.backupId = backupId; + } + + public String getVmName() { + return vmName; + } + + public Long getVmId() { + return vmId; + } + + public Long getBackupId() { + return backupId; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StopNBDServerCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StopNBDServerCommand.java new file mode 100644 index 000000000000..d75168a22eb2 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StopNBDServerCommand.java @@ -0,0 +1,46 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; + +public class StopNBDServerCommand extends Command { + private String transferId; + private String direction; + + public StopNBDServerCommand() { + } + + public StopNBDServerCommand(String transferId, String direction) { + this.transferId = transferId; + this.direction = direction; + } + + public String getTransferId() { + return transferId; + } + + public String getDirection() { + return direction; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/debian/control b/debian/control index 2b8ce929c639..cdf663ef8906 100644 --- a/debian/control +++ b/debian/control @@ -24,7 +24,7 @@ Description: CloudStack server library Package: cloudstack-agent Architecture: all -Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat +Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, socat Recommends: init-system-helpers Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts Description: CloudStack agent diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java index e871bd8672ff..5eca02f33d51 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java @@ -59,6 +59,8 @@ */ public interface VirtualMachineManager extends Manager { + String KVM_BLANK_VM_TEMPLATE_NAME = "kvm-blank-vm-template"; + ConfigKey ExecuteInSequence = new ConfigKey<>("Advanced", Boolean.class, "execute.in.sequence.hypervisor.commands", "false", "If set to true, start, stop, reboot, copy and migrate commands will be serialized on the agent side. If set to false the commands are executed in parallel. Default value is false.", false); @@ -312,4 +314,8 @@ void checkDeploymentPlan(VirtualMachine virtualMachine, VirtualMachineTemplate t ServiceOffering serviceOffering, Account systemAccount, DeploymentPlan plan) throws InsufficientServerCapacityException; + boolean isBlankInstanceDefaultTemplate(VirtualMachineTemplate template); + + boolean isBlankInstance(VirtualMachineTemplate template); + } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index ead990b42b86..db50a5134d8d 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -303,8 +303,8 @@ import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.UserVmDao; -import com.cloud.vm.dao.VMInstanceDetailsDao; import com.cloud.vm.dao.VMInstanceDao; +import com.cloud.vm.dao.VMInstanceDetailsDao; import com.cloud.vm.snapshot.VMSnapshotManager; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; @@ -576,7 +576,13 @@ public void allocate(final String vmInstanceName, final VirtualMachineTemplate t logger.debug("Allocating disks for {}", persistedVm); - allocateRootVolume(persistedVm, template, rootDiskOfferingInfo, owner, rootDiskSizeFinal, volume, snapshot); + if (isBlankInstance(template)) { + logger.debug("Template is a dummy template for hypervisor {}, skipping volume allocation", hyperType); + return; + } else { + allocateRootVolume(persistedVm, template, rootDiskOfferingInfo, owner, rootDiskSizeFinal, volume, snapshot); + } + // Create new Volume context and inject event resource type, id and details to generate VOLUME.CREATE event for the ROOT disk. CallContext volumeContext = CallContext.register(CallContext.current(), ApiCommandResourceType.Volume); @@ -6708,4 +6714,18 @@ public void checkDeploymentPlan(VirtualMachine virtualMachine, VirtualMachineTem vmProfile), DataCenter.class, plan.getDataCenterId(), areAffinityGroupsAssociated(vmProfile)); } } + + @Override + public boolean isBlankInstanceDefaultTemplate(VirtualMachineTemplate template) { + return KVM_BLANK_VM_TEMPLATE_NAME.equals(template.getUniqueName()); + } + + @Override + public boolean isBlankInstance(VirtualMachineTemplate template) { + if (isBlankInstanceDefaultTemplate(template)) { + return true; + } + return Boolean.TRUE.equals( + MapUtils.getBoolean(CallContext.current().getContextParameters(), ApiConstants.BLANK_INSTANCE)); + } } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java index 8639f006383f..964265cb873e 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.engine.orchestration; import com.cloud.storage.Snapshot; +import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.template.VirtualMachineTemplate; import java.net.URL; @@ -292,9 +293,11 @@ public VirtualMachineEntity createVirtualMachineFromScratch(String id, String ow ServiceOfferingVO computeOffering = _serviceOfferingDao.findById(vm.getId(), vm.getServiceOfferingId()); + VMTemplateVO iso = _templateDao.findByIdIncludingRemoved(Long.valueOf(isoId)); + DiskOfferingInfo rootDiskOfferingInfo = new DiskOfferingInfo(); - if (diskOfferingId == null) { + if (diskOfferingId == null && !_itMgr.isBlankInstance(iso)) { throw new InvalidParameterValueException("Installing from ISO requires a disk offering to be specified for the root disk."); } DiskOfferingVO diskOffering = _diskOfferingDao.findById(diskOfferingId); @@ -345,7 +348,7 @@ public VirtualMachineEntity createVirtualMachineFromScratch(String id, String ow HypervisorType hypervisorType = HypervisorType.valueOf(hypervisor); - _itMgr.allocate(vm.getInstanceName(), _templateDao.findByIdIncludingRemoved(new Long(isoId)), computeOffering, rootDiskOfferingInfo, dataDiskOfferings, dataDiskDeviceIds, + _itMgr.allocate(vm.getInstanceName(), iso, computeOffering, rootDiskOfferingInfo, dataDiskOfferings, dataDiskDeviceIds, networkIpMap, plan, hypervisorType, extraDhcpOptionMap, null, volume, snapshot); return vmEntity; diff --git a/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml b/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml index 17c5002c718b..49c668f50e8b 100644 --- a/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml +++ b/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml @@ -88,6 +88,7 @@ + diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java index 6cfd2608f5de..3d04f6679def 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java @@ -23,6 +23,7 @@ import com.cloud.dc.ClusterVO; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; public interface ClusterDao extends GenericDao { @@ -61,4 +62,6 @@ public interface ClusterDao extends GenericDao { List listDistinctStorageAccessGroups(String name, String keyword); List listEnabledClusterIdsByZoneHypervisorArch(Long zoneId, HypervisorType hypervisorType, CPU.CPUArch arch); + + List listByZonesAndHypervisorType(List zoneIds, HypervisorType hypervisorType, Filter filter); } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java index c63af0a237ba..9b6f0f7ac3a6 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java @@ -20,6 +20,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -27,6 +28,7 @@ import javax.inject.Inject; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.cpu.CPU; @@ -38,6 +40,7 @@ import com.cloud.org.Grouping; import com.cloud.org.Managed; import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.JoinBuilder; @@ -413,4 +416,19 @@ public List listEnabledClusterIdsByZoneHypervisorArch(Long zoneId, Hypervi } return customSearch(sc, null); } + + @Override + public List listByZonesAndHypervisorType(List zoneIds, HypervisorType hypervisorType, Filter filter) { + if (CollectionUtils.isEmpty(zoneIds)) { + return Collections.emptyList(); + } + SearchBuilder sb = createSearchBuilder(); + sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.IN); + sb.and("hypervisorType", sb.entity().getHypervisorType(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("dataCenterId", zoneIds.toArray()); + sc.setParameters("hypervisorType", hypervisorType.toString()); + return listBy(sc, filter); + } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java index fdca6e43f00f..8406f80ed09d 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java @@ -24,6 +24,7 @@ import com.cloud.network.Network.GuestType; import com.cloud.network.Network.State; import com.cloud.network.Networks.TrafficType; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.fsm.StateDao; @@ -96,8 +97,13 @@ public interface NetworkDao extends GenericDao, StateDao serviceProviderMap); + List listByZoneAndTrafficType(long zoneId, TrafficType trafficType, Filter filter); + List listByZoneAndTrafficType(long zoneId, TrafficType trafficType); + List listByZonesTrafficTypeAndOwners(List zoneIds, final TrafficType trafficType, + List accountIds, List domainIds, Filter filter); + void setCheckForGc(long networkId); int getNetworkCountByNetworkOffId(long networkOfferingId); diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java index 9f7ffabac930..f3ece4879040 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java @@ -19,6 +19,7 @@ import java.net.URI; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -29,9 +30,9 @@ import javax.inject.Inject; import javax.persistence.TableGenerator; -import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.api.ApiConstants; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.network.Network; @@ -63,6 +64,7 @@ import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.SequenceFetcher; import com.cloud.utils.db.TransactionLegacy; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; @Component @@ -632,12 +634,46 @@ public List listBy(final long accountId, final long dataCenterId, fin } @Override - public List listByZoneAndTrafficType(final long zoneId, final TrafficType trafficType) { + public List listByZoneAndTrafficType(final long zoneId, final TrafficType trafficType, Filter filter) { final SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("datacenter", zoneId); sc.setParameters("trafficType", trafficType); - return listBy(sc, null); + return listBy(sc, filter); + } + + @Override + public List listByZoneAndTrafficType(final long zoneId, final TrafficType trafficType) { + return listByZoneAndTrafficType(zoneId, trafficType, null); + } + + @Override + public List listByZonesTrafficTypeAndOwners(List zoneIds, final TrafficType trafficType, + List accountIds, List domainIds, Filter filter) { + if (CollectionUtils.isEmpty(zoneIds)) { + return Collections.emptyList(); + } + SearchBuilder sb = createSearchBuilder(); + sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.IN); + sb.and("trafficType", sb.entity().getTrafficType(), Op.EQ); + boolean accountIdsNotEmpty = CollectionUtils.isNotEmpty(accountIds); + boolean domainIdsNotEmpty = CollectionUtils.isNotEmpty(domainIds); + if (accountIdsNotEmpty || domainIdsNotEmpty) { + sb.and().op("account", sb.entity().getAccountId(), SearchCriteria.Op.IN); + sb.or("domain", sb.entity().getDomainId(), SearchCriteria.Op.IN); + sb.cp(); + } + sb.done(); + final SearchCriteria sc = sb.create(); + sc.setParameters("dataCenterId", zoneIds.toArray()); + sc.setParameters("trafficType", trafficType); + if (accountIdsNotEmpty) { + sc.setParameters("account", accountIds.toArray()); + } + if (domainIdsNotEmpty) { + sc.setParameters("domain", domainIds.toArray()); + } + return listBy(sc, filter); } @Override diff --git a/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDao.java b/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDao.java index 793334731095..2b18bcbd87bd 100644 --- a/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDao.java +++ b/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDao.java @@ -31,4 +31,6 @@ public interface SecurityGroupDao extends GenericDao { List findByAccountAndNames(Long accountId, String... names); int removeByAccountId(long accountId); + + List listByIds(List ids); } diff --git a/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDaoImpl.java index 019cf5fec462..ffb45629b625 100644 --- a/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDaoImpl.java @@ -129,4 +129,14 @@ public boolean expunge(Long id) { txn.commit(); return result; } + + @Override + public List listByIds(List ids) { + SearchBuilder idsSearch = createSearchBuilder(); + idsSearch.and("ids", idsSearch.entity().getId(), SearchCriteria.Op.IN); + idsSearch.done(); + SearchCriteria sc = idsSearch.create(); + sc.setParameters("ids", ids.toArray()); + return listBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java index 9beea0037444..ea97f663768f 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java @@ -30,7 +30,7 @@ public interface DiskOfferingDao extends GenericDao { List listAllBySizeAndProvisioningType(long size, Storage.ProvisioningType provisioningType); - List findCustomDiskOfferings(); + List listCustomDiskOfferings(); List listByStorageTag(String tag); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java index 4ca3fe9f12ac..44e6551da17d 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.storage.dao; +import static org.apache.cloudstack.query.QueryService.SortKeyAscending; + import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -33,6 +35,7 @@ import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.Storage; import com.cloud.utils.db.Attribute; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -138,13 +141,14 @@ public List listAllBySizeAndProvisioningType(long size, Storage. } @Override - public List findCustomDiskOfferings() { + public List listCustomDiskOfferings() { SearchBuilder sb = createSearchBuilder(); sb.and("customized", sb.entity().isCustomized(), SearchCriteria.Op.EQ); sb.done(); SearchCriteria sc = sb.create(); sc.setParameters("customized", true); - return listBy(sc); + Filter searchFilter = new Filter(DiskOfferingVO.class, "sortKey", SortKeyAscending.value()); + return listBy(sc, searchFilter); } @Override diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java index 4c9f906b68a9..aec06d6d0003 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java @@ -106,4 +106,6 @@ public interface VMTemplateDao extends GenericDao, StateDao< VMTemplateVO findActiveSystemTemplateByHypervisorArchAndUrlPath(HypervisorType hypervisorType, CPU.CPUArch arch, String urlPathSuffix); + + VMTemplateVO findByAccountAndName(Long accountId, String templateName); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index 9b5d0edc599d..8c6e3fe0983f 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -945,4 +945,12 @@ public boolean updateState( } return rows > 0; } + + @Override + public VMTemplateVO findByAccountAndName(Long accountId, String templateName) { + SearchCriteria sc = NameAccountIdSearch.create(); + sc.setParameters("name", templateName); + sc.setParameters("accountId", accountId); + return findOneBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java index 717e3e782f2f..7d566a1ea01e 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java @@ -46,6 +46,8 @@ public interface VolumeDao extends GenericDao, StateDao findByInstanceAndType(long id, Volume.Type vType); + List findByInstanceAndNotStates(long id, Volume.State...states); + List findIncludingRemovedByInstanceAndType(long id, Volume.Type vType); List findNonDestroyedVolumesByInstanceIdAndPoolId(long instanceId, long poolId); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java index fce4d1f7233d..91f1c7f5eb69 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java @@ -208,6 +208,17 @@ public List findByInstanceAndType(long id, Type vType) { return listBy(sc); } + @Override + public List findByInstanceAndNotStates(long id, Volume.State...states) { + SearchBuilder sb = createSearchBuilder(); + sb.and("instanceId", sb.entity().getInstanceId(), Op.EQ); + sb.and("state", sb.entity().getState(), Op.NIN); + SearchCriteria sc = sb.create(); + sc.setParameters("instanceId", id); + sc.setParameters("state", (Object[]) states); + return listBy(sc); + } + @Override public List findIncludingRemovedByInstanceAndType(long id, Type vType) { SearchCriteria sc = AllFieldsSearch.create(); diff --git a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java index bacb09b98793..5efaea40a943 100644 --- a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java +++ b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java @@ -20,11 +20,13 @@ import java.util.Map; import java.util.Set; +import org.apache.cloudstack.api.response.ResourceTagResponse; + import com.cloud.server.ResourceTag; import com.cloud.server.ResourceTag.ResourceObjectType; import com.cloud.tags.ResourceTagVO; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; -import org.apache.cloudstack.api.response.ResourceTagResponse; public interface ResourceTagDao extends GenericDao { @@ -60,4 +62,13 @@ public interface ResourceTagDao extends GenericDao { void removeByResourceIdAndKey(long resourceId, ResourceObjectType resourceType, String key); List listByResourceUuid(String resourceUuid); + + List listByResourceTypeKeyPrefixAndOwners(ResourceObjectType resourceType, String key, + List accountIds, List domainIds, + Filter filter); + + ResourceTagVO findByResourceTypeKeyPrefixAndValue(ResourceObjectType resourceType, String key, String value); + + List listByResourceTypeIdAndKeyPrefix(ResourceObjectType resourceType, long resourceId, String key); + } diff --git a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java index cc9d99e6ab16..22c7b7b2ee5b 100644 --- a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java @@ -16,19 +16,22 @@ // under the License. package com.cloud.tags.dao; -import java.util.List; -import java.util.Set; -import java.util.Map; import java.util.HashMap; import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.cloudstack.api.response.ResourceTagResponse; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.server.ResourceTag; import com.cloud.server.ResourceTag.ResourceObjectType; import com.cloud.tags.ResourceTagVO; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; @@ -120,4 +123,62 @@ public List listByResourceUuid(String resourceUuid) { sc.setParameters("resourceUuid", resourceUuid); return listBy(sc); } + + @Override + public List listByResourceTypeKeyPrefixAndOwners(ResourceObjectType resourceType, String key, + List accountIds, List domainIds, + Filter filter) { + GenericSearchBuilder sb = createSearchBuilder(String.class); + sb.select(null, SearchCriteria.Func.DISTINCT, sb.entity().getValue()); + sb.and("resourceType", sb.entity().getResourceType(), Op.EQ); + sb.and("key", sb.entity().getKey(), Op.LIKE); + boolean accountIdsNotEmpty = CollectionUtils.isNotEmpty(accountIds); + boolean domainIdsNotEmpty = CollectionUtils.isNotEmpty(domainIds); + if (accountIdsNotEmpty || domainIdsNotEmpty) { + sb.and().op("account", sb.entity().getAccountId(), SearchCriteria.Op.IN); + sb.or("domain", sb.entity().getDomainId(), SearchCriteria.Op.IN); + sb.cp(); + } + sb.done(); + final SearchCriteria sc = sb.create(); + sc.setParameters("resourceType", resourceType); + sc.setParameters("key", key + "%"); + if (accountIdsNotEmpty) { + sc.setParameters("account", accountIds.toArray()); + } + if (domainIdsNotEmpty) { + sc.setParameters("domain", domainIds.toArray()); + } + return customSearch(sc, filter); + } + + @Override + public ResourceTagVO findByResourceTypeKeyPrefixAndValue(ResourceObjectType resourceType, String key, + String value) { + SearchBuilder sb = createSearchBuilder(); + sb.and("resourceType", sb.entity().getResourceType(), Op.EQ); + sb.and("key", sb.entity().getKey(), Op.LIKE); + sb.and("value", sb.entity().getValue(), Op.EQ); + sb.done(); + final SearchCriteria sc = sb.create(); + sc.setParameters("resourceType", resourceType); + sc.setParameters("key", key + "%"); + sc.setParameters("value", value); + return findOneBy(sc); + } + + @Override + public List listByResourceTypeIdAndKeyPrefix(ResourceObjectType resourceType, long resourceId, + String key) { + SearchBuilder sb = createSearchBuilder(); + sb.and("resourceType", sb.entity().getResourceType(), Op.EQ); + sb.and("resourceId", sb.entity().getResourceId(), Op.EQ); + sb.and("key", sb.entity().getKey(), Op.LIKE); + sb.done(); + final SearchCriteria sc = sb.create(); + sc.setParameters("resourceType", resourceType); + sc.setParameters("resourceId", resourceId); + sc.setParameters("key", key + "%"); + return listBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java index 96a1b6e3bd16..7767b7512b91 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java @@ -26,6 +26,8 @@ public interface NicDao extends GenericDao { List listByVmId(long instanceId); + int countByVmId(long instanceId); + List listByVmIdOrderByDeviceId(long instanceId); List listIpAddressInNetwork(long networkConfigId); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java index 78966a09e97c..6202dabf730e 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java @@ -126,6 +126,13 @@ public List listByVmId(long instanceId) { return listBy(sc); } + @Override + public int countByVmId(long instanceId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("instance", instanceId); + return getCount(sc); + } + @Override public List listByVmIdOrderByDeviceId(long instanceId) { SearchCriteria sc = AllFieldsSearch.create(); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java index 4fd3e729e0d2..1a5b8cedd9ea 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java @@ -197,4 +197,6 @@ List searchRemovedByRemoveDate(final Date startDate, final Date en List listDeleteProtectedVmsByAccountId(long accountId); List listDeleteProtectedVmsByDomainIds(Set domainIds); + + List listIdsByHostIdForVolumeStats(long hostIds); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index 019d152ce5c3..ae1e838649ba 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -1336,4 +1336,20 @@ public List listDeleteProtectedVmsByDomainIds(Set domainIds) Filter filter = new Filter(VMInstanceVO.class, null, false, 0L, 10L); return listBy(sc, filter); } + + @Override + public List listIdsByHostIdForVolumeStats(long hostId) { + GenericSearchBuilder sb = createSearchBuilder(Long.class); + sb.selectFields(sb.entity().getId()); + sb.and().op("host", sb.entity().getHostId(), SearchCriteria.Op.EQ); + sb.or().op("hostNull", sb.entity().getHostId(), Op.NULL); + sb.and("lastHost", sb.entity().getLastHostId(), SearchCriteria.Op.EQ); + sb.cp(); + sb.cp(); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("host", hostId); + sc.setParameters("lastHost", hostId); + return customSearch(sc, null); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java index 0f8a10fb7be6..d589f9e6bef8 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java @@ -103,6 +103,18 @@ public class BackupVO implements Backup { @Column(name = "backup_schedule_id") private Long backupScheduleId; + @Column(name = "from_checkpoint_id") + private String fromCheckpointId; + + @Column(name = "to_checkpoint_id") + private String toCheckpointId; + + @Column(name = "checkpoint_create_time") + private Long checkpointCreateTime; + + @Column(name = "host_id") + private Long hostId; + @Transient Map details; @@ -288,4 +300,40 @@ public Long getBackupScheduleId() { public void setBackupScheduleId(Long backupScheduleId) { this.backupScheduleId = backupScheduleId; } + + @Override + public String getFromCheckpointId() { + return fromCheckpointId; + } + + public void setFromCheckpointId(String fromCheckpointId) { + this.fromCheckpointId = fromCheckpointId; + } + + @Override + public String getToCheckpointId() { + return toCheckpointId; + } + + public void setToCheckpointId(String toCheckpointId) { + this.toCheckpointId = toCheckpointId; + } + + @Override + public Long getCheckpointCreateTime() { + return checkpointCreateTime; + } + + public void setCheckpointCreateTime(Long checkpointCreateTime) { + this.checkpointCreateTime = checkpointCreateTime; + } + + @Override + public Long getHostId() { + return hostId; + } + + public void setHostId(Long hostId) { + this.hostId = hostId; + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java new file mode 100644 index 000000000000..ec9b927b63e4 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java @@ -0,0 +1,242 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +@Entity +@Table(name = "image_transfer") +public class ImageTransferVO implements ImageTransfer { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "uuid") + private String uuid; + + @Column(name = "backup_id") + private Long backupId; + + @Column(name = "volume_id") + private long volumeId; + + @Column(name = "host_id") + private long hostId; + + @Column(name = "socket") + private String socket; + + @Column(name = "file") + private String file; + + @Column(name = "transfer_url") + private String transferUrl; + + @Enumerated(value = EnumType.STRING) + @Column(name = "phase") + private Phase phase; + + @Enumerated(value = EnumType.STRING) + @Column(name = "direction") + private Direction direction; + + @Enumerated(value = EnumType.STRING) + @Column(name = "backend") + private Backend backend; + + @Column(name = "signed_ticket_id") + private String signedTicketId; + + @Column(name = "account_id") + Long accountId; + + @Column(name = "domain_id") + Long domainId; + + @Column(name = "data_center_id") + Long dataCenterId; + + @Column(name = "created") + @Temporal(value = TemporalType.TIMESTAMP) + private Date created; + + @Column(name = "updated") + @Temporal(value = TemporalType.TIMESTAMP) + private Date updated; + + @Column(name = "removed") + @Temporal(value = TemporalType.TIMESTAMP) + private Date removed; + + public ImageTransferVO() { + } + + private ImageTransferVO(String uuid, long volumeId, long hostId, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { + this.uuid = uuid; + this.volumeId = volumeId; + this.hostId = hostId; + this.phase = phase; + this.direction = direction; + this.accountId = accountId; + this.domainId = domainId; + this.dataCenterId = dataCenterId; + this.created = new Date(); + } + + public ImageTransferVO(String uuid, Long backupId, long volumeId, long hostId, String socket, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { + this(uuid, volumeId, hostId, phase, direction, accountId, domainId, dataCenterId); + this.backupId = backupId; + this.socket = socket; + this.backend = Backend.nbd; + } + + public ImageTransferVO(String uuid, long volumeId, long hostId, String file, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { + this(uuid, volumeId, hostId, phase, direction, accountId, domainId, dataCenterId); + this.file = file; + this.backend = Backend.file; + } + + @Override + public long getId() { + return id; + } + + @Override + public String getUuid() { + return uuid; + } + + @Override + public Long getBackupId() { + return backupId; + } + + public void setBackupId(long backupId) { + this.backupId = backupId; + } + + @Override + public long getVolumeId() { + return volumeId; + } + + public void setVolumeId(long volumeId) { + this.volumeId = volumeId; + } + + @Override + public long getHostId() { + return hostId; + } + + public void setHostId(long hostId) { + this.hostId = hostId; + } + + public void setSocket(String socket) { + this.socket = socket; + } + + @Override + public String getTransferUrl() { + return transferUrl; + } + + public void setTransferUrl(String transferUrl) { + this.transferUrl = transferUrl; + } + + @Override + public Phase getPhase() { + return phase; + } + + public void setPhase(Phase phase) { + this.phase = phase; + this.updated = new Date(); + } + + @Override + public Direction getDirection() { + return direction; + } + + public void setDirection(Direction direction) { + this.direction = direction; + } + + @Override + public Backend getBackend() { + return backend; + } + + @Override + public String getSignedTicketId() { + return signedTicketId; + } + + public void setSignedTicketId(String signedTicketId) { + this.signedTicketId = signedTicketId; + } + + @Override + public Class getEntityType() { + return ImageTransfer.class; + } + + @Override + public String getName() { + return null; + } + + @Override + public long getDomainId() { + return domainId; + } + + @Override + public long getAccountId() { + return accountId; + } + + @Override + public long getDataCenterId() { + return dataCenterId; + } + + public Date getCreated() { + return created; + } + + public Date getUpdated() { + return updated; + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java new file mode 100644 index 000000000000..9a9fbeb8814e --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java @@ -0,0 +1,36 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup.dao; + +import java.util.List; + +import org.apache.cloudstack.backup.ImageTransfer; +import org.apache.cloudstack.backup.ImageTransferVO; + +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDao; + +public interface ImageTransferDao extends GenericDao { + List listByBackupId(Long backupId); + ImageTransferVO findByUuid(String uuid); + ImageTransferVO findByVolume(Long volumeId); + ImageTransferVO findUnfinishedByVolume(Long volumeId); + List listByPhaseAndDirection(ImageTransfer.Phase phase, ImageTransfer.Direction direction); + List listByZonesAndOwners(List zoneIds, List accountIds, List domainIds, + Filter filter); +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java new file mode 100644 index 000000000000..2e36f924e4ef --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java @@ -0,0 +1,136 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup.dao; + +import java.util.Collections; +import java.util.List; + +import javax.annotation.PostConstruct; + +import org.apache.cloudstack.backup.ImageTransfer; +import org.apache.cloudstack.backup.ImageTransferVO; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.stereotype.Component; + +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@Component +public class ImageTransferDaoImpl extends GenericDaoBase implements ImageTransferDao { + + private SearchBuilder backupIdSearch; + private SearchBuilder uuidSearch; + private SearchBuilder volumeSearch; + private SearchBuilder volumeUnfinishedSearch; + private SearchBuilder phaseDirectionSearch; + + public ImageTransferDaoImpl() { + } + + @PostConstruct + protected void init() { + backupIdSearch = createSearchBuilder(); + backupIdSearch.and("backupId", backupIdSearch.entity().getBackupId(), SearchCriteria.Op.EQ); + backupIdSearch.done(); + + uuidSearch = createSearchBuilder(); + uuidSearch.and("uuid", uuidSearch.entity().getUuid(), SearchCriteria.Op.EQ); + uuidSearch.done(); + + volumeSearch = createSearchBuilder(); + volumeSearch.and("volumeId", volumeSearch.entity().getVolumeId(), SearchCriteria.Op.EQ); + volumeSearch.done(); + + volumeUnfinishedSearch = createSearchBuilder(); + volumeUnfinishedSearch.and("volumeId", volumeUnfinishedSearch.entity().getVolumeId(), SearchCriteria.Op.EQ); + volumeUnfinishedSearch.and("phase", volumeUnfinishedSearch.entity().getPhase(), SearchCriteria.Op.NEQ); + volumeUnfinishedSearch.done(); + + phaseDirectionSearch = createSearchBuilder(); + phaseDirectionSearch.and("phase", phaseDirectionSearch.entity().getPhase(), SearchCriteria.Op.EQ); + phaseDirectionSearch.and("direction", phaseDirectionSearch.entity().getDirection(), SearchCriteria.Op.EQ); + phaseDirectionSearch.done(); + } + + @Override + public List listByBackupId(Long backupId) { + SearchCriteria sc = backupIdSearch.create(); + sc.setParameters("backupId", backupId); + return listBy(sc); + } + + @Override + public ImageTransferVO findByUuid(String uuid) { + SearchCriteria sc = uuidSearch.create(); + sc.setParameters("uuid", uuid); + return findOneBy(sc); + } + + @Override + public ImageTransferVO findByVolume(Long volumeId) { + SearchCriteria sc = volumeSearch.create(); + sc.setParameters("volumeId", volumeId); + return findOneBy(sc); + } + + @Override + public ImageTransferVO findUnfinishedByVolume(Long volumeId) { + SearchCriteria sc = volumeUnfinishedSearch.create(); + sc.setParameters("volumeId", volumeId); + sc.setParameters("phase", ImageTransferVO.Phase.finished.toString()); + return findOneBy(sc); + } + + @Override + public List listByPhaseAndDirection(ImageTransfer.Phase phase, ImageTransfer.Direction direction) { + SearchCriteria sc = phaseDirectionSearch.create(); + sc.setParameters("phase", phase); + sc.setParameters("direction", direction); + return listBy(sc); + } + + @Override + public List listByZonesAndOwners(List zoneIds, List accountIds, List domainIds, + Filter filter) { + if (CollectionUtils.isEmpty(zoneIds)) { + return Collections.emptyList(); + } + SearchBuilder sb = createSearchBuilder(); + sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.IN); + boolean accountIdsNotEmpty = CollectionUtils.isNotEmpty(accountIds); + boolean domainIdsNotEmpty = CollectionUtils.isNotEmpty(domainIds); + if (accountIdsNotEmpty || domainIdsNotEmpty) { + sb.and().op("account", sb.entity().getAccountId(), SearchCriteria.Op.IN); + sb.or("domain", sb.entity().getDomainId(), SearchCriteria.Op.IN); + sb.cp(); + } + sb.done(); + final SearchCriteria sc = sb.create(); + sc.setParameters("dataCenterId", zoneIds.toArray()); + if (accountIdsNotEmpty) { + sc.setParameters("account", accountIds.toArray()); + } + if (domainIdsNotEmpty) { + sc.setParameters("domain", domainIds.toArray()); + } + + return listBy(sc, filter); + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDao.java index 4735202a7623..82ba9445b25c 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDao.java @@ -29,4 +29,6 @@ public interface SharedFSDao extends GenericDao, StateDao listSharedFSToBeDestroyed(Date date); SharedFSVO findSharedFSByNameAccountDomain(String name, Long accountId, Long domainId); + + SharedFSVO findByVm(long vmId); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDaoImpl.java index da6220716715..dd23787f982f 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDaoImpl.java @@ -114,4 +114,14 @@ public SharedFSVO findSharedFSByNameAccountDomain(String name, Long accountId, L sc.setParameters("domainId", domainId); return findOneBy(sc); } + + @Override + public SharedFSVO findByVm(long vmId) { + SearchBuilder sb = createSearchBuilder(); + sb.and("vmId", sb.entity().getVmId(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("vmId", vmId); + return findOneBy(sc); + } } diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index ad3722577c27..26181d3fce0d 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -116,6 +116,7 @@ + @@ -271,6 +272,7 @@ + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql index 858c46a7c1ee..fbb2fd079f9c 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql @@ -19,7 +19,6 @@ -- Schema upgrade from 4.21.0.0 to 4.22.0.0 --; - -- health check status as enum CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('router_health_check', 'check_result', 'check_result', 'varchar(16) NOT NULL COMMENT "check executions result: SUCCESS, FAILURE, WARNING, UNKNOWN"'); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index dfa5fa8f3a14..478125335e70 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -149,3 +149,50 @@ SELECT 'Advanced', 'DEFAULT', 'CapacityManager', 'kvm.cpu.dynamic.scaling.capaci FROM `cloud`.`configuration` `cfg` WHERE NOT EXISTS (SELECT 1 FROM `cloud`.`configuration` WHERE `name` = 'kvm.cpu.dynamic.scaling.capacity') AND `cfg`.`name` = 'vm.serviceoffering.cpu.cores.max'; + +-- Add management_server_details table to allow ManagementServer scope configs +CREATE TABLE IF NOT EXISTS `management_server_details` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `management_server_id` bigint unsigned NOT NULL COMMENT 'management server the detail is related to', + `name` varchar(255) NOT NULL COMMENT 'name of the detail', + `value` varchar(255) NOT NULL, + `display` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'True if the detail can be displayed to the end user', + PRIMARY KEY (`id`), + CONSTRAINT `fk_management_server_details__management_server_id` FOREIGN KEY `fk_management_server_details__management_server_id`(`management_server_id`) REFERENCES `mshost`(`id`) ON DELETE CASCADE, + KEY `i_management_server_details__name__value` (`name`(128),`value`(128)) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- Add checkpoint tracking fields to backups table for incremental backup support +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'from_checkpoint_id', 'VARCHAR(255) DEFAULT NULL COMMENT "Previous active checkpoint id for incremental backups"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'to_checkpoint_id', 'VARCHAR(255) DEFAULT NULL COMMENT "New checkpoint id created for the next incremental backup"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'checkpoint_create_time', 'BIGINT DEFAULT NULL COMMENT "Checkpoint creation timestamp from libvirt"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'host_id', 'BIGINT UNSIGNED DEFAULT NULL COMMENT "Host where backup is running"'); + +-- Create image_transfer table for per-disk image transfers +CREATE TABLE IF NOT EXISTS `cloud`.`image_transfer`( + `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', + `uuid` varchar(40) NOT NULL COMMENT 'uuid', + `account_id` bigint unsigned NOT NULL COMMENT 'Account ID', + `domain_id` bigint unsigned NOT NULL COMMENT 'Domain ID', + `data_center_id` bigint unsigned NOT NULL COMMENT 'Data Center ID', + `backup_id` bigint unsigned COMMENT 'Backup ID', + `volume_id` bigint unsigned NOT NULL COMMENT 'Volume ID', + `host_id` bigint unsigned NOT NULL COMMENT 'Host ID', + `transfer_url` varchar(255) COMMENT 'ImageIO transfer URL', + `file` varchar(255) COMMENT 'File for the file backend', + `phase` varchar(20) NOT NULL COMMENT 'Transfer phase: initializing, transferring, finished, failed', + `socket` varchar(255) COMMENT 'Unix socket for nbd backend', + `direction` varchar(20) NOT NULL COMMENT 'Direction: upload, download', + `backend` varchar(20) NOT NULL COMMENT 'Backend: nbd, file', + `progress` int COMMENT 'Transfer progress percentage (0-100)', + `signed_ticket_id` varchar(255) COMMENT 'Signed ticket ID from ImageIO', + `created` datetime NOT NULL COMMENT 'date created', + `updated` datetime COMMENT 'date updated if not null', + `removed` datetime COMMENT 'date removed if not null', + PRIMARY KEY (`id`), + UNIQUE KEY `uuid` (`uuid`), + CONSTRAINT `fk_image_transfer__backup_id` FOREIGN KEY (`backup_id`) REFERENCES `backups`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_image_transfer__volume_id` FOREIGN KEY (`volume_id`) REFERENCES `volumes`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_image_transfer__host_id` FOREIGN KEY (`host_id`) REFERENCES `host`(`id`) ON DELETE CASCADE, + INDEX `i_image_transfer__backup_id`(`backup_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql index c169cbce2170..d77ba27689be 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql @@ -56,6 +56,7 @@ SELECT `vm_instance`.`display_vm` AS `display_vm`, `vm_instance`.`delete_protection` AS `delete_protection`, `guest_os`.`uuid` AS `guest_os_uuid`, + `guest_os`.`display_name` AS `guest_os_display_name`, `vm_instance`.`pod_id` AS `pod_id`, `host_pod_ref`.`uuid` AS `pod_uuid`, `vm_instance`.`private_ip_address` AS `private_ip_address`, diff --git a/engine/schema/src/test/java/com/cloud/storage/dao/VolumeDaoImplTest.java b/engine/schema/src/test/java/com/cloud/storage/dao/VolumeDaoImplTest.java index 9445efeb089c..6f153727ab76 100644 --- a/engine/schema/src/test/java/com/cloud/storage/dao/VolumeDaoImplTest.java +++ b/engine/schema/src/test/java/com/cloud/storage/dao/VolumeDaoImplTest.java @@ -41,6 +41,7 @@ import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.utils.db.Filter; import com.cloud.utils.db.SearchBuilder; @@ -113,6 +114,69 @@ public void testListPoolIdsByVolumeCount_without_cluster_details() throws SQLExc verify(preparedStatementMock, times(1)).executeQuery(); } + @Test + public void findByInstanceAndNotState_queriesWithInstanceIdAndExcludedStates() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doReturn(new ArrayList<>()).when(volumeDao).listBy(sc); + Mockito.when(volumeDao.createSearchBuilder()).thenReturn(sb); + VolumeVO mockedVO = Mockito.mock(VolumeVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + + volumeDao.findByInstanceAndNotStates(42L, Volume.State.Ready); + + Mockito.verify(sc).setParameters("instanceId", 42L); + Mockito.verify(sc).setParameters("state", (Object[]) new Volume.State[]{Volume.State.Ready}); + } + + @Test + public void findByInstanceAndNotStates_withMultipleExcludedStates_passesAllStatesToCriteria() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doReturn(new ArrayList<>()).when(volumeDao).listBy(sc); + Mockito.when(volumeDao.createSearchBuilder()).thenReturn(sb); + VolumeVO mockedVO = Mockito.mock(VolumeVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + + volumeDao.findByInstanceAndNotStates(7L, Volume.State.Destroy, Volume.State.Expunged); + + Mockito.verify(sc).setParameters("instanceId", 7L); + Mockito.verify(sc).setParameters("state", + (Object[]) new Volume.State[]{Volume.State.Destroy, Volume.State.Expunged}); + } + + @Test + public void findByInstanceAndNotStates_returnsResultFromDao() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + VolumeVO vol = Mockito.mock(VolumeVO.class); + Mockito.doReturn(List.of(vol)).when(volumeDao).listBy(sc); + Mockito.when(volumeDao.createSearchBuilder()).thenReturn(sb); + Mockito.when(sb.entity()).thenReturn(Mockito.mock(VolumeVO.class)); + + List result = volumeDao.findByInstanceAndNotStates(1L, Volume.State.Ready); + + Assert.assertEquals(1, result.size()); + Assert.assertSame(vol, result.get(0)); + } + + @Test + public void findByInstanceAndNotStates_noMatchingVolumes_returnsEmptyList() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doReturn(new ArrayList<>()).when(volumeDao).listBy(sc); + Mockito.when(volumeDao.createSearchBuilder()).thenReturn(sb); + Mockito.when(sb.entity()).thenReturn(Mockito.mock(VolumeVO.class)); + + List result = volumeDao.findByInstanceAndNotStates(99L, Volume.State.Ready); + + Assert.assertTrue(result.isEmpty()); + } + @Test public void testSearchRemovedByVmsNoVms() { Assert.assertTrue(CollectionUtils.isEmpty(volumeDao.searchRemovedByVms( @@ -141,5 +205,4 @@ public void testSearchRemovedByVms() { Mockito.any(SearchCriteria.class), Mockito.any(Filter.class), Mockito.eq(null), Mockito.eq(false)); } - } diff --git a/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java b/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java index 7c5692564c99..c9c48dbb179f 100644 --- a/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java +++ b/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java @@ -119,10 +119,10 @@ public String validateUserData(String userData, BaseCmd.HTTPMethod httpmethod) { byte[] decodedUserData = null; // If GET, use 4K. If POST, support up to 1M. - if (httpmethod.equals(BaseCmd.HTTPMethod.GET)) { - decodedUserData = validateAndDecodeByHTTPMethod(userData, MAX_HTTP_GET_LENGTH, BaseCmd.HTTPMethod.GET); - } else if (httpmethod.equals(BaseCmd.HTTPMethod.POST)) { + if (BaseCmd.HTTPMethod.POST.equals(httpmethod)) { decodedUserData = validateAndDecodeByHTTPMethod(userData, MAX_HTTP_POST_LENGTH, BaseCmd.HTTPMethod.POST); + } else { + decodedUserData = validateAndDecodeByHTTPMethod(userData, MAX_HTTP_GET_LENGTH, BaseCmd.HTTPMethod.GET); } // Re-encode so that the '=' paddings are added if necessary since 'isBase64' does not require it, but python does on the VR. diff --git a/framework/cluster/pom.xml b/framework/cluster/pom.xml index 2dd28e8e628f..75bcaf9a7ddf 100644 --- a/framework/cluster/pom.xml +++ b/framework/cluster/pom.xml @@ -48,6 +48,12 @@ cloud-api ${project.version} + + org.apache.cloudstack + cloud-engine-schema + ${project.version} + compile + diff --git a/framework/cluster/src/main/java/com/cloud/cluster/ManagementServerHostDetailVO.java b/framework/cluster/src/main/java/com/cloud/cluster/ManagementServerHostDetailVO.java new file mode 100644 index 000000000000..fcaa2a22e341 --- /dev/null +++ b/framework/cluster/src/main/java/com/cloud/cluster/ManagementServerHostDetailVO.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.cluster; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.apache.cloudstack.api.ResourceDetail; + +@Entity +@Table(name = "management_server_details") +public class ManagementServerHostDetailVO implements ResourceDetail { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + long id; + + @Column(name = "management_server_id") + long resourceId; + + @Column(name = "name") + String name; + + @Column(name = "value") + String value; + + @Column(name = "display") + private boolean display = true; + + public ManagementServerHostDetailVO(long poolId, String name, String value, boolean display) { + this.resourceId = poolId; + this.name = name; + this.value = value; + this.display = display; + } + + public ManagementServerHostDetailVO() { + } + + @Override + public long getId() { + return id; + } + + @Override + public long getResourceId() { + return resourceId; + } + + @Override + public String getName() { + return name; + } + + public void setValue(String value) { + this.value = value; + } + + @Override + public String getValue() { + return value; + } + + @Override + public boolean isDisplay() { + return display; + } +} diff --git a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDao.java b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDao.java new file mode 100644 index 000000000000..24fd60d21b3c --- /dev/null +++ b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDao.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.cluster.dao; + +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; + +import com.cloud.cluster.ManagementServerHostDetailVO; +import com.cloud.utils.db.GenericDao; + +public interface ManagementServerHostDetailsDao extends GenericDao, ResourceDetailsDao { +} diff --git a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDaoImpl.java b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDaoImpl.java new file mode 100644 index 000000000000..5865bee0926b --- /dev/null +++ b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDaoImpl.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.cluster.dao; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.ScopedConfigStorage; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; + +import com.cloud.cluster.ManagementServerHostDetailVO; + +public class ManagementServerHostDetailsDaoImpl extends ResourceDetailsDaoBase implements ManagementServerHostDetailsDao, ScopedConfigStorage { + + public ManagementServerHostDetailsDaoImpl() { + } + + @Override + public ConfigKey.Scope getScope() { + return ConfigKey.Scope.ManagementServer; + } + + @Override + public String getConfigValue(long id, String key) { + ManagementServerHostDetailVO vo = findDetail(id, key); + return vo == null ? null : vo.getValue(); + } + + @Override + public void addDetail(long resourceId, String key, String value, boolean display) { + super.addDetail(new ManagementServerHostDetailVO(resourceId, key, value, display)); + } +} diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java index 926280bfeade..c334e91feb84 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java @@ -68,4 +68,6 @@ public interface AsyncJobDao extends GenericDao { // Returns the number of pending jobs for the given Management server msids. // NOTE: This is the msid and NOT the id long countPendingNonPseudoJobs(Long... msIds); + + List listPendingJobIdsForAccount(long accountId); } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java index 81cc5d4f2a8c..d8385e9aecd1 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java @@ -299,4 +299,14 @@ public long countPendingJobs(String havingInfo, String... cmds) { List results = customSearch(sc, null); return results.get(0); } + + @Override + public List listPendingJobIdsForAccount(long accountId) { + GenericSearchBuilder sb = createSearchBuilder(Long.class); + sb.and("accountId", sb.entity().getAccountId(), SearchCriteria.Op.EQ); + sb.selectFields(sb.entity().getId()); + SearchCriteria sc = sb.create(); + sc.setParameters("accountId", accountId); + return customSearch(sc, null); + } } diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index 13c0a36cb106..3ba2e4d5789e 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -127,6 +127,8 @@ Requires: rng-tools Requires: (libgcrypt > 1.8.3 or libgcrypt20) Requires: (selinux-tools if selinux-tools) Requires: sysstat +Requires: python3-libnbd +Requires: socat Provides: cloud-agent Group: System Environment/Libraries %description agent diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/ImageServerControlSocket.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/ImageServerControlSocket.java new file mode 100644 index 000000000000..2e9852f7bc1e --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/ImageServerControlSocket.java @@ -0,0 +1,123 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +/** + * Communicates with the cloudstack-image-server control socket via socat. + * + * Protocol: newline-delimited JSON over a Unix domain socket. + * Actions: register, unregister, status. + */ +public class ImageServerControlSocket { + private static final Logger LOGGER = LogManager.getLogger(ImageServerControlSocket.class); + static final String CONTROL_SOCKET_PATH = "/var/run/cloudstack/image-server.sock"; + private static final Gson GSON = new GsonBuilder().create(); + + private ImageServerControlSocket() { + } + + /** + * Send a JSON message to the image server control socket and return the + * parsed response, or null on communication failure. + */ + static JsonObject sendMessage(Map message) { + String json = GSON.toJson(message); + Script script = new Script("/bin/bash", LOGGER); + script.add("-c"); + script.add(String.format("echo '%s' | socat -t5 - UNIX-CONNECT:%s", + json.replace("'", "'\\''"), CONTROL_SOCKET_PATH)); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + String result = script.execute(parser); + if (result != null) { + LOGGER.error("Control socket communication failed: {}", result); + return null; + } + String output = parser.getLines(); + if (output == null || output.trim().isEmpty()) { + LOGGER.error("Empty response from control socket"); + return null; + } + try { + return JsonParser.parseString(output.trim()).getAsJsonObject(); + } catch (Exception e) { + LOGGER.error("Failed to parse control socket response: {}", output, e); + return null; + } + } + + /** + * Register a transfer config with the image server. + * @return true if the server accepted the registration. + */ + public static boolean registerTransfer(String transferId, Map config) { + Map msg = new HashMap<>(); + msg.put("action", "register"); + msg.put("transfer_id", transferId); + msg.put("config", config); + JsonObject resp = sendMessage(msg); + if (resp == null) { + return false; + } + return "ok".equals(resp.has("status") ? resp.get("status").getAsString() : null); + } + + /** + * Unregister a transfer from the image server. + * @return the number of remaining active transfers, or -1 on error. + */ + public static int unregisterTransfer(String transferId) { + Map msg = new HashMap<>(); + msg.put("action", "unregister"); + msg.put("transfer_id", transferId); + JsonObject resp = sendMessage(msg); + if (resp == null) { + return -1; + } + if (!"ok".equals(resp.has("status") ? resp.get("status").getAsString() : null)) { + return -1; + } + return resp.has("active_transfers") ? resp.get("active_transfers").getAsInt() : -1; + } + + /** + * Check whether the image server control socket is responsive. + * @return true if the server responded with status "ok". + */ + public static boolean isReady() { + Map msg = new HashMap<>(); + msg.put("action", "status"); + JsonObject resp = sendMessage(msg); + if (resp == null) { + return false; + } + return "ok".equals(resp.has("status") ? resp.get("status").getAsString() : null); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 06b668b801d3..d0a162e0f6ff 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -387,6 +387,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public static final String CHECKPOINT_DELETE_COMMAND = "virsh checkpoint-delete --domain %s --checkpointname %s --metadata"; + public static final int IMAGE_SERVER_DEFAULT_PORT = 54322; + public static final String IMAGE_SERVER_SYSTEMD_UNIT_NAME = "cloudstack-image-server"; + protected int qcow2DeltaMergeTimeout; private String modifyVlanPath; @@ -400,6 +403,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private String heartBeatPath; private String vmActivityCheckPath; private String nasBackupPath; + private String imageServerPath; + private boolean imageServerTlsEnabled = false; + private String imageServerListenAddress; private String securityGroupPath; private String ovsPvlanDhcpHostPath; private String ovsPvlanVmPath; @@ -814,6 +820,18 @@ public String getNasBackupPath() { return nasBackupPath; } + public String getImageServerPath() { + return imageServerPath; + } + + public boolean isImageServerTlsEnabled() { + return imageServerTlsEnabled; + } + + public String getImageServerListenAddress() { + return imageServerListenAddress; + } + public String getOvsPvlanDhcpHostPath() { return ovsPvlanDhcpHostPath; } @@ -1077,6 +1095,9 @@ public boolean configure(final String name, final Map params) th cachePath = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_CACHE_LOCATION); + imageServerTlsEnabled = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.IMAGE_SERVER_TLS_ENABLED); + imageServerListenAddress = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.IMAGE_SERVER_LISTEN_ADDRESS); + params.put("domr.scripts.dir", domrScriptsDir); virtRouterResource = new VirtualRoutingResource(this); @@ -1140,6 +1161,12 @@ public boolean configure(final String name, final Map params) th throw new ConfigurationException("Unable to find nasbackup.sh"); } + String imageServerMain = Script.findScript(kvmScriptsDir, "imageserver/__main__.py"); + if (imageServerMain == null) { + throw new ConfigurationException("Unable to find imageserver package"); + } + imageServerPath = new File(imageServerMain).getParent(); + createTmplPath = Script.findScript(storageScriptsDir, "createtmplt.sh"); if (createTmplPath == null) { throw new ConfigurationException("Unable to find the createtmplt.sh"); @@ -5349,6 +5376,24 @@ public void removeCheckpointsOnVm(String vmName, String volumeUuid, List logger.debug("Removed all checkpoints of volume [{}] on VM [{}].", volumeUuid, vmName); } + public Map getDiskPathLabelMap(String vmName) { + try { + Connect conn = LibvirtConnection.getConnectionByVmName(vmName); + List disks = getDisks(conn, vmName); + Map diskPathLabelMap = new HashMap<>(); + for (DiskDef disk : disks) { + if (disk.getDeviceType() != DeviceType.DISK) { + continue; + } + diskPathLabelMap.put(disk.getDiskPath(), disk.getDiskLabel()); + } + return diskPathLabelMap; + } catch (LibvirtException e) { + logger.error("Failed to get disk path label map for VM [{}] due to: [{}].", vmName, e.getMessage(), e); + throw new CloudRuntimeException(e); + } + } + public boolean recreateCheckpointsOnVm(List volumes, String vmName, Connect conn) { logger.debug("Trying to recreate checkpoints on VM [{}] with volumes [{}].", vmName, volumes); try { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java new file mode 100644 index 000000000000..7cf05da9b211 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java @@ -0,0 +1,178 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +import org.apache.cloudstack.backup.CreateImageTransferAnswer; +import org.apache.cloudstack.backup.CreateImageTransferCommand; +import org.apache.cloudstack.backup.ImageTransfer; +import org.apache.cloudstack.storage.resource.IpTablesHelper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.ImageServerControlSocket; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.StringUtils; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = CreateImageTransferCommand.class) +public class LibvirtCreateImageTransferCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + private static final String IMAGE_SERVER_TLS_CERT_FILE = "/etc/cloudstack/agent/cloud.crt"; + private static final String IMAGE_SERVER_TLS_KEY_FILE = "/etc/cloudstack/agent/cloud.key"; + + private void resetService(String unitName) { + Script resetScript = new Script("/bin/bash", logger); + resetScript.add("-c"); + resetScript.add(String.format("systemctl reset-failed %s || true", unitName)); + resetScript.execute(); + } + + private static String shellQuote(String value) { + return "'" + value.replace("'", "'\\''") + "'"; + } + + private boolean startImageServerIfNotRunning(int imageServerPort, String listenAddress, LibvirtComputingResource resource) { + final String imageServerPackageDir = resource.getImageServerPath(); + final String imageServerParentDir = new File(imageServerPackageDir).getParent(); + final String imageServerModuleName = new File(imageServerPackageDir).getName(); + final boolean tlsEnabled = resource.isImageServerTlsEnabled(); + String unitName = resource.IMAGE_SERVER_SYSTEMD_UNIT_NAME; + + Script checkScript = new Script("/bin/bash", logger); + checkScript.add("-c"); + checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String checkResult = checkScript.execute(); + if (checkResult == null && ImageServerControlSocket.isReady()) { + return true; + } + + resetService(unitName); + if (checkResult != null) { + StringBuilder systemdRunCmd = new StringBuilder(String.format( + "systemd-run --unit=%s --property=Restart=no --property=WorkingDirectory=%s /usr/bin/python3 -m %s --listen %s --port %d", + unitName, shellQuote(imageServerParentDir), imageServerModuleName, shellQuote(listenAddress), imageServerPort)); + + if (tlsEnabled) { + systemdRunCmd.append(" --tls-enabled"); + systemdRunCmd.append(" --tls-cert-file ").append(IMAGE_SERVER_TLS_CERT_FILE); + systemdRunCmd.append(" --tls-key-file ").append(IMAGE_SERVER_TLS_KEY_FILE); + } + + Script startScript = new Script("/bin/bash", logger); + startScript.add("-c"); + startScript.add(systemdRunCmd.toString()); + String startResult = startScript.execute(); + + if (startResult != null) { + logger.error(String.format("Failed to start the Image server: %s", startResult)); + return false; + } + } + + int maxWaitSeconds = 10; + int pollIntervalMs = 1000; + int maxAttempts = (maxWaitSeconds * 1000) / pollIntervalMs; + boolean serverReady = false; + + for (int attempt = 0; attempt < maxAttempts; attempt++) { + if (ImageServerControlSocket.isReady()) { + serverReady = true; + logger.info(String.format("Image server control socket is ready (attempt %d)", attempt + 1)); + break; + } + try { + Thread.sleep(pollIntervalMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + + if (!serverReady) { + logger.error(String.format("Image server control socket not ready within %d seconds", maxWaitSeconds)); + return false; + } + + String rule = String.format("-p tcp -m state --state NEW -m tcp --dport %d -j ACCEPT", imageServerPort); + IpTablesHelper.addConditionally(IpTablesHelper.INPUT_CHAIN, true, rule, + String.format("Error in opening up image server port %d", imageServerPort)); + + return true; + } + + public Answer execute(CreateImageTransferCommand cmd, LibvirtComputingResource resource) { + final String transferId = cmd.getTransferId(); + ImageTransfer.Backend backend = cmd.getBackend(); + + if (StringUtils.isBlank(transferId)) { + return new CreateImageTransferAnswer(cmd, false, "transferId is empty."); + } + + final Map payload = new HashMap<>(); + payload.put("backend", backend.toString()); + payload.put("idle_timeout_seconds", cmd.getIdleTimeoutSeconds()); + + if (backend == ImageTransfer.Backend.file) { + final String filePath = cmd.getFile(); + if (StringUtils.isBlank(filePath)) { + return new CreateImageTransferAnswer(cmd, false, "file path is empty for file backend."); + } + payload.put("file", filePath); + } else { + String socket = cmd.getSocket(); + final String exportName = cmd.getExportName(); + if (StringUtils.isBlank(socket)) { + return new CreateImageTransferAnswer(cmd, false, "Empty socket."); + } + if (StringUtils.isBlank(exportName)) { + return new CreateImageTransferAnswer(cmd, false, "exportName is empty."); + } + payload.put("socket", "/tmp/imagetransfer/" + socket + ".sock"); + payload.put("export", exportName); + String checkpointId = cmd.getCheckpointId(); + if (checkpointId != null) { + payload.put("export_bitmap", cmd.getCheckpointId()); + } + } + + final int imageServerPort = LibvirtComputingResource.IMAGE_SERVER_DEFAULT_PORT; + String listenAddress = resource.getImageServerListenAddress(); + if (StringUtils.isBlank(listenAddress)) { + listenAddress = resource.getPrivateIp(); + } + if (!startImageServerIfNotRunning(imageServerPort, listenAddress, resource)) { + return new CreateImageTransferAnswer(cmd, false, "Failed to start image server."); + } + + if (!ImageServerControlSocket.registerTransfer(transferId, payload)) { + return new CreateImageTransferAnswer(cmd, false, "Failed to register transfer with image server."); + } + + final String transferScheme = resource.isImageServerTlsEnabled() ? "https" : "http"; + final String transferUrl = String.format("%s://%s:%d/images/%s", transferScheme, listenAddress, imageServerPort, transferId); + return new CreateImageTransferAnswer(cmd, true, "Image transfer prepared on KVM host.", transferId, transferUrl); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapper.java new file mode 100644 index 000000000000..ddb84ab29cb3 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapper.java @@ -0,0 +1,79 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.util.Map; + +import org.apache.cloudstack.backup.DeleteVmCheckpointCommand; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = DeleteVmCheckpointCommand.class) +public class LibvirtDeleteVmCheckpointCommandWrapper extends CommandWrapper { + + @Override + public Answer execute(DeleteVmCheckpointCommand cmd, LibvirtComputingResource resource) { + if (cmd.isStoppedVM()) { + return deleteBitmapsOnDisks(cmd); + } + return deleteDomainCheckpoint(cmd); + } + + private Answer deleteDomainCheckpoint(DeleteVmCheckpointCommand cmd) { + String vmName = cmd.getVmName(); + String checkpointId = cmd.getCheckpointId(); + String virshCmd = String.format("virsh checkpoint-delete %s %s", vmName, checkpointId); + Script script = new Script("/bin/bash"); + script.add("-c"); + script.add(virshCmd); + String result = script.execute(); + if (result != null) { + return new Answer(cmd, false, "Failed to delete checkpoint: " + result); + } + return new Answer(cmd, true, "Checkpoint deleted"); + } + + /** + * Stopped VM: persistent bitmaps on disk images ({@code qemu-img bitmap --remove}), matching {@link LibvirtStartBackupCommandWrapper} bitmap --add. + */ + private Answer deleteBitmapsOnDisks(DeleteVmCheckpointCommand cmd) { + String checkpointId = cmd.getCheckpointId(); + Map diskPathUuidMap = cmd.getDiskPathUuidMap(); + if (diskPathUuidMap == null || diskPathUuidMap.isEmpty()) { + return new Answer(cmd, false, "No disks provided for bitmap removal"); + } + for (Map.Entry entry : diskPathUuidMap.entrySet()) { + String diskPath = entry.getKey(); + Script script = new Script("qemu-img"); + script.add("bitmap"); + script.add("--remove"); + script.add(diskPath); + script.add(checkpointId); + String result = script.execute(); + if (result != null) { + return new Answer(cmd, false, + "Failed to remove bitmap " + checkpointId + " from disk " + diskPath + ": " + result); + } + } + return new Answer(cmd, true, "Checkpoint bitmap removed from disks"); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java new file mode 100644 index 000000000000..3d9f6563d5eb --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java @@ -0,0 +1,101 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import org.apache.cloudstack.backup.FinalizeImageTransferCommand; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.ImageServerControlSocket; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.StringUtils; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = FinalizeImageTransferCommand.class) +public class LibvirtFinalizeImageTransferCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + private void resetService(String unitName) { + Script resetScript = new Script("/bin/bash", logger); + resetScript.add("-c"); + resetScript.add(String.format("systemctl reset-failed %s || true", unitName)); + resetScript.execute(); + } + + private boolean stopImageServer(int imageServerPort, LibvirtComputingResource resource) { + String unitName = resource.IMAGE_SERVER_SYSTEMD_UNIT_NAME; + + Script checkScript = new Script("/bin/bash", logger); + checkScript.add("-c"); + checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String checkResult = checkScript.execute(); + if (checkResult != null) { + logger.info("Image server not running, resetting failed state"); + resetService(unitName); + removeFirewallRule(imageServerPort); + return true; + } + + Script stopScript = new Script("/bin/bash", logger); + stopScript.add("-c"); + stopScript.add(String.format("systemctl stop %s", unitName)); + stopScript.execute(); + resetService(unitName); + logger.info("Image server {} stopped", unitName); + + removeFirewallRule(imageServerPort); + + return true; + } + + private void removeFirewallRule(int port) { + String rule = String.format("-p tcp -m state --state NEW -m tcp --dport %d -j ACCEPT", port); + Script removeScript = new Script("/bin/bash", logger); + removeScript.add("-c"); + removeScript.add(String.format("iptables -D INPUT %s || true", rule)); + String result = removeScript.execute(); + if (result != null && !result.isEmpty() && !result.contains("iptables: Bad rule")) { + logger.debug("Firewall rule removal result for port {}: {}", port, result); + } else { + logger.info("Firewall rule removed for port {} (or did not exist)", port); + } + } + + public Answer execute(FinalizeImageTransferCommand cmd, LibvirtComputingResource resource) { + final String transferId = cmd.getTransferId(); + final int imageServerPort = LibvirtComputingResource.IMAGE_SERVER_DEFAULT_PORT; + if (StringUtils.isBlank(transferId)) { + return new Answer(cmd, false, "transferId is empty."); + } + + int activeTransfers = ImageServerControlSocket.unregisterTransfer(transferId); + if (activeTransfers < 0) { + logger.warn("Could not reach image server to unregister transfer {}; assuming server is down", transferId); + stopImageServer(imageServerPort, resource); + return new Answer(cmd, true, "Image transfer finalized (server unreachable, forced stop)."); + } + + if (activeTransfers == 0) { + stopImageServer(imageServerPort, resource); + } + + return new Answer(cmd, true, "Image transfer finalized."); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java new file mode 100644 index 000000000000..629fd179afb2 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java @@ -0,0 +1,300 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.cloudstack.backup.StartBackupAnswer; +import org.apache.cloudstack.backup.StartBackupCommand; +import org.apache.cloudstack.utils.cryptsetup.KeyFile; +import org.apache.cloudstack.utils.qemu.QemuCommand; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.json.JSONArray; +import org.json.JSONObject; +import org.libvirt.Domain; +import org.libvirt.LibvirtException; +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.StringUtils; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = StartBackupCommand.class) +public class LibvirtStartBackupCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + @Override + public Answer execute(StartBackupCommand cmd, LibvirtComputingResource resource) { + if (cmd.isStoppedVM()) { + return handleStoppedVmBackup(cmd, cmd.getToCheckpointId()); + } + return handleRunningVmBackup(cmd, resource); + } + + public Answer handleRunningVmBackup(StartBackupCommand cmd, LibvirtComputingResource resource) { + String vmName = cmd.getVmName(); + String toCheckpointId = cmd.getToCheckpointId(); + String fromCheckpointId = cmd.getFromCheckpointId(); + Long fromCheckpointCreateTime = cmd.getFromCheckpointCreateTime(); + String socket = cmd.getSocket(); + + try { + if (StringUtils.isNotBlank(fromCheckpointId)) { + Answer redefineAnswer = ensureFromCheckpointExists(cmd, fromCheckpointId, fromCheckpointCreateTime); + if (redefineAnswer != null) { + return redefineAnswer; + } + } + + File dir = new File("/tmp/imagetransfer"); + if (!dir.exists()) { + dir.mkdirs(); + } + + // Create backup XML + String backupXml = createBackupXml(cmd, fromCheckpointId, socket, resource); + String checkpointXml = createCheckpointXml(toCheckpointId); + + // Write XMLs to temp files + File backupXmlFile = File.createTempFile("backup-", ".xml"); + File checkpointXmlFile = File.createTempFile("checkpoint-", ".xml"); + + try (FileWriter writer = new FileWriter(backupXmlFile)) { + writer.write(backupXml); + } + try (FileWriter writer = new FileWriter(checkpointXmlFile)) { + writer.write(checkpointXml); + } + + // Execute virsh backup-begin + String backupCmd = String.format("virsh backup-begin %s %s --checkpointxml %s", + vmName, backupXmlFile.getAbsolutePath(), checkpointXmlFile.getAbsolutePath()); + + Script script = new Script("/bin/bash"); + script.add("-c"); + script.add(backupCmd); + String result = script.execute(); + + backupXmlFile.delete(); + checkpointXmlFile.delete(); + + if (result != null) { + return new StartBackupAnswer(cmd, false, "Backup begin failed: " + result); + } + + long checkpointCreateTime = getCheckpointCreateTime(); + return new StartBackupAnswer(cmd, true, "Backup started successfully", checkpointCreateTime); + + } catch (Exception e) { + return new StartBackupAnswer(cmd, false, "Error starting backup: " + e.getMessage()); + } + } + + private Answer ensureFromCheckpointExists(StartBackupCommand cmd, String fromCheckpointId, Long fromCheckpointCreateTime) { + String vmName = cmd.getVmName(); + Script dumpScript = new Script("/bin/bash"); + dumpScript.add("-c"); + dumpScript.add(String.format("virsh checkpoint-dumpxml --domain %s --checkpointname %s --no-domain", + vmName, fromCheckpointId)); + if (dumpScript.execute() == null) { + return null; + } + if (fromCheckpointCreateTime == null) { + return new StartBackupAnswer(cmd, false, "From checkpoint create time is null for checkpoint " + fromCheckpointId); + } + + String redefineXml = createCheckpointXmlForRedefine(fromCheckpointId, fromCheckpointCreateTime); + File redefineFile; + try { + redefineFile = File.createTempFile("checkpoint-redefine-", ".xml"); + } catch (Exception e) { + return new StartBackupAnswer(cmd, false, "Failed to create temp file for checkpoint redefine: " + e.getMessage()); + } + try (FileWriter writer = new FileWriter(redefineFile)) { + writer.write(redefineXml); + } catch (Exception e) { + redefineFile.delete(); + return new StartBackupAnswer(cmd, false, "Failed to write checkpoint redefine XML: " + e.getMessage()); + } + String createCmd = String.format(LibvirtComputingResource.CHECKPOINT_CREATE_COMMAND, vmName, redefineFile.getAbsolutePath()); + Script createScript = new Script("/bin/bash"); + createScript.add("-c"); + createScript.add(createCmd); + String result = createScript.execute(); + redefineFile.delete(); + if (result != null) { + return new StartBackupAnswer(cmd, false, "Failed to redefine from-checkpoint " + fromCheckpointId + ": " + result); + } + return null; + } + + private String createCheckpointXmlForRedefine(String checkpointName, Long createTime) { + StringBuilder xml = new StringBuilder(); + xml.append("\n"); + xml.append(" ").append(checkpointName).append("\n"); + xml.append(" ").append(createTime).append("\n"); + xml.append(""); + return xml.toString(); + } + + private String createBackupXml(StartBackupCommand cmd, String fromCheckpointId, String socket, LibvirtComputingResource resource) throws LibvirtException { + StringBuilder xml = new StringBuilder(); + xml.append("\n"); + + xml.append(String.format(" \n", socket)); + + xml.append(" \n"); + + Map diskPathUuidMap = cmd.getDiskPathUuidMap(); + Map diskPathLabelMap = resource.getDiskPathLabelMap(cmd.getVmName()); + Map diskPathHasFromCheckpointMap = new HashMap<>(); + if (StringUtils.isNotBlank(fromCheckpointId)) { + Domain vm = null; + try { + vm = resource.getDomain(resource.getLibvirtUtilitiesHelper().getConnection(), cmd.getVmName()); + if (vm != null) { + diskPathHasFromCheckpointMap = getVmDiskPathHasFromCheckpointMap(vm, fromCheckpointId); + } else { + logger.warn("Failed to get domain for VM [{}] while evaluating export bitmap [{}]. Falling back to full Backup", + cmd.getVmName(), fromCheckpointId); + } + } finally { + if (vm != null) { + vm.free(); + } + } + } + + for (Map.Entry entry : diskPathLabelMap.entrySet()) { + String diskPath = entry.getKey(); + if (!diskPathUuidMap.containsKey(diskPath)) { + continue; + } + String diskName = entry.getValue(); + String export = diskPathUuidMap.get(diskPath); + String scratchFile = "/var/tmp/scratch-" + export + ".qcow2"; + xml.append(" \n"); + xml.append(" \n"); + xml.append(" \n"); + } + + xml.append(" \n"); + xml.append(""); + + return xml.toString(); + } + + private String createCheckpointXml(String checkpointId) { + return "\n" + + " " + checkpointId + "\n" + + ""; + } + + private Answer handleStoppedVmBackup(StartBackupCommand cmd, String toCheckpointId) { + Map diskPathUuidMap = cmd.getDiskPathUuidMap(); + Map diskPathPassphraseMap = cmd.getDiskPathPassphraseMap(); + for (Map.Entry entry : diskPathUuidMap.entrySet()) { + String diskPath = entry.getKey(); + Script script = new Script("qemu-img"); + byte[] passphrase = diskPathPassphraseMap.get(diskPath); + + script.add("bitmap"); + script.add("--add"); + + if (passphrase != null && passphrase.length > 0) { + KeyFile srcKey; + try { + srcKey = new KeyFile(passphrase); + } catch (IOException ex) { + return new StartBackupAnswer(cmd, false, "Failed to create KeyFile while adding bitmap " + toCheckpointId + " to disk " + diskPath); + } + script.add("--object"); + script.add(String.format("secret,id=sec0,file=%s", srcKey)); + script.add("--image-opts"); + script.add(String.format("driver=qcow2,file.driver=file,file.filename=%s,encrypt.key-secret=sec0", diskPath)); + } else { + script.add(diskPath); + } + + script.add(toCheckpointId); + String result = script.execute(); + if (result != null) { + return new StartBackupAnswer(cmd, false, + "Failed to add bitmap " + toCheckpointId + " to disk " + diskPath + ": " + result); + } + } + long checkpointCreateTime = getCheckpointCreateTime(); + return new StartBackupAnswer(cmd, true, "Stopped VM backup: checkpoint bitmap added successfully", + checkpointCreateTime); + } + + private long getCheckpointCreateTime() { + return System.currentTimeMillis() / 1000; + } + + private Map getVmDiskPathHasFromCheckpointMap(Domain vm, String fromCheckpointId) throws LibvirtException { + Map diskPathHasFromCheckpointMap = new HashMap<>(); + String queryBlock = vm.qemuMonitorCommand(QemuCommand.buildQemuCommand("query-block", null), 0); + JSONObject response = new JSONObject(queryBlock); + JSONArray blocks = response.optJSONArray("return"); + if (blocks == null) { + logger.warn("Couldn't get bitmap information for the VM [{}]. Falling back to full Backup", vm.getName()); + return diskPathHasFromCheckpointMap; + } + for (int i = 0; i < blocks.length(); i++) { + JSONObject block = blocks.getJSONObject(i); + JSONObject inserted = block.optJSONObject("inserted"); + if (inserted == null) { + continue; + } + String file = inserted.optString("file"); + if (StringUtils.isBlank(file)) { + continue; + } + JSONArray dirtyBitmaps = inserted.optJSONArray("dirty-bitmaps"); + boolean hasFromCheckpointBitmap = false; + if (dirtyBitmaps != null) { + for (int j = 0; j < dirtyBitmaps.length(); j++) { + JSONObject dirtyBitmap = dirtyBitmaps.optJSONObject(j); + if (dirtyBitmap == null) { + continue; + } + String bitmapName = dirtyBitmap.optString("name"); + if (fromCheckpointId.equals(bitmapName)) { + hasFromCheckpointBitmap = true; + break; + } + } + } + diskPathHasFromCheckpointMap.put(file, hasFromCheckpointBitmap); + } + return diskPathHasFromCheckpointMap; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java new file mode 100644 index 000000000000..661344677640 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java @@ -0,0 +1,197 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.io.File; +import java.io.IOException; + +import org.apache.cloudstack.backup.StartNBDServerAnswer; +import org.apache.cloudstack.backup.StartNBDServerCommand; +import org.apache.cloudstack.utils.cryptsetup.KeyFile; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.json.JSONArray; +import org.json.JSONObject; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.StringUtils; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = StartNBDServerCommand.class) +public class LibvirtStartNBDServerCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + private String createSystemdRunCmd(StartNBDServerCommand cmd, String unitName, String volumePath, String exportName, String socket) throws IOException { + String socketName = "/tmp/imagetransfer/" + socket + ".sock"; + + String bitmapArg = ""; + if (StringUtils.isNotBlank(cmd.getFromCheckpointId()) + && isBitmapPresentOnDisk(volumePath, cmd.getFromCheckpointId())) { + bitmapArg = "-B " + cmd.getFromCheckpointId(); + } + + byte[] passphrase = cmd.getPassphrase(); // may be null + String readOnlyArg = cmd.getDirection().equals("download") ? "--read-only" : ""; + String imageArg = volumePath; + String secretArg = ""; + if (passphrase != null && passphrase.length > 0) { + KeyFile srcKey = new KeyFile(passphrase); + secretArg = String.format("--object secret,id=sec0,file=%s", srcKey); + imageArg = String.format("--image-opts driver=qcow2,file.driver=file,file.filename=%s,encrypt.key-secret=sec0", volumePath); + } + + // --persistent: Don't stop the service when the last client disconnects. + // --shared=NUM: Allow up to NUM clients to share the device (default 1), 0 for unlimited. Number of parallel connections is managed by the image server. + String systemdRunCmd = String.format( + "systemd-run --unit=%s --property=Restart=no qemu-nbd %s " + + "--export-name %s --socket %s --persistent --shared=0 %s %s %s", + unitName, + secretArg, + exportName, + socketName, + bitmapArg, + readOnlyArg, + imageArg + ); + return systemdRunCmd; + } + + @Override + public Answer execute(StartNBDServerCommand cmd, LibvirtComputingResource resource) { + String volumePath = cmd.getVolumePath(); + String socket = cmd.getSocket(); + String exportName = cmd.getExportName(); + String transferId = cmd.getTransferId(); + + if (StringUtils.isBlank(volumePath)) { + return new StartNBDServerAnswer(cmd, false, "Volume path is required for the nbd server"); + } + if (StringUtils.isBlank(exportName)) { + return new StartNBDServerAnswer(cmd, false, "Export name is required for the nbd server"); + } + if (StringUtils.isBlank(socket)) { + return new StartNBDServerAnswer(cmd, false, "Socket is required for the nbd server"); + } + + String unitName = "qemu-nbd-" + transferId.hashCode(); + + Script checkScript = new Script("/bin/bash", logger); + checkScript.add("-c"); + checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String checkResult = checkScript.execute(); + if (checkResult == null) { + return new StartNBDServerAnswer(cmd, false, "A qemu-nbd service is already running on the port."); + } + + File dir = new File("/tmp/imagetransfer"); + if (!dir.exists()) { + dir.mkdirs(); + } + + String systemdRunCmd = ""; + try { + systemdRunCmd = createSystemdRunCmd(cmd, unitName, volumePath, exportName, socket); + } catch (IOException e) { + logger.error("Failed to create the KeyFile for qemu-nbd service", e); + return new StartNBDServerAnswer(cmd, false, "Failed to create systemd run command for qemu-nbd service: " + e.getMessage()); + } + + Script startScript = new Script("/bin/bash", logger); + startScript.add("-c"); + startScript.add(systemdRunCmd); + String startResult = startScript.execute(); + + if (startResult != null) { + logger.error(String.format("Failed to start qemu-nbd service: %s", startResult)); + return new StartNBDServerAnswer(cmd, false, "Failed to start qemu-nbd service: " + startResult); + } + + // Wait with timeout until the service is up + int maxWaitSeconds = 20; + int pollIntervalMs = 5000; + int maxAttempts = (maxWaitSeconds * 1000) / pollIntervalMs; + boolean serviceActive = false; + + for (int attempt = 0; attempt < maxAttempts; attempt++) { + try { + Thread.sleep(pollIntervalMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return new StartNBDServerAnswer(cmd, false, "Interrupted while waiting for qemu-nbd service to start"); + } + Script verifyScript = new Script("/bin/bash", logger); + verifyScript.add("-c"); + verifyScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String verifyResult = verifyScript.execute(); + if (verifyResult == null) { + serviceActive = true; + logger.info(String.format("qemu-nbd service %s is now active (attempt %d)", unitName, attempt + 1)); + break; + } + } + + if (!serviceActive) { + logger.error(String.format("qemu-nbd service %s failed to become active within %d seconds", unitName, maxWaitSeconds)); + return new StartNBDServerAnswer(cmd, false, + String.format("qemu-nbd service failed to start within %d seconds", maxWaitSeconds)); + } + + String transferUrl = String.format("nbd+unix:///%s", cmd.getSocket()); + return new StartNBDServerAnswer(cmd, true, "qemu-nbd service started for upload", + transferId, transferUrl); + } + + private boolean isBitmapPresentOnDisk(String volumePath, String fromCheckpointId) { + String qemuImgInfo = Script.runBashScriptIgnoreExitValue( + String.format("qemu-img info --output=json %s", volumePath), 0); + if (StringUtils.isBlank(qemuImgInfo)) { + logger.warn("Unable to read qemu-img info output for disk path [{}].", volumePath); + return false; + } + try { + JSONObject info = new JSONObject(qemuImgInfo); + JSONObject formatSpecific = info.optJSONObject("format-specific"); + if (formatSpecific == null) { + return false; + } + JSONObject formatData = formatSpecific.optJSONObject("data"); + if (formatData == null) { + return false; + } + JSONArray bitmaps = formatData.optJSONArray("bitmaps"); + if (bitmaps == null) { + return false; + } + for (int i = 0; i < bitmaps.length(); i++) { + JSONObject bitmap = bitmaps.optJSONObject(i); + if (bitmap == null) { + continue; + } + if (fromCheckpointId.equals(bitmap.optString("name"))) { + return true; + } + } + } catch (Exception e) { + logger.warn("Failed to parse qemu-img info output for disk path [{}].", volumePath, e); + } + return false; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopBackupCommandWrapper.java new file mode 100644 index 000000000000..1185d89bc0b3 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopBackupCommandWrapper.java @@ -0,0 +1,69 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import org.apache.cloudstack.backup.StopBackupAnswer; +import org.apache.cloudstack.backup.StopBackupCommand; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.libvirt.Connect; +import org.libvirt.Domain; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.resource.LibvirtConnection; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = StopBackupCommand.class) +public class LibvirtStopBackupCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + @Override + public Answer execute(StopBackupCommand cmd, LibvirtComputingResource resource) { + String vmName = cmd.getVmName(); + + try { + Connect conn = LibvirtConnection.getConnection(); + Domain dm = conn.domainLookupByName(vmName); + + if (dm == null) { + return new StopBackupAnswer(cmd, false, "Domain not found: " + vmName); + } + + // Execute virsh domjobabort + String abortCmd = String.format("virsh domjobabort %s", vmName); + + Script script = new Script("/bin/bash"); + script.add("-c"); + script.add(abortCmd); + String result = script.execute(); + + if (result != null && !result.isEmpty()) { + // Job abort may fail if no job is running, which is acceptable + logger.debug("domjobabort result: " + result); + } + + return new StopBackupAnswer(cmd, true, "Backup stopped successfully"); + + } catch (Exception e) { + return new StopBackupAnswer(cmd, false, "Error stopping backup: " + e.getMessage()); + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopNBDServerCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopNBDServerCommandWrapper.java new file mode 100644 index 000000000000..57c7ebb706bc --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopNBDServerCommandWrapper.java @@ -0,0 +1,72 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import org.apache.cloudstack.backup.StopNBDServerCommand; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = StopNBDServerCommand.class) +public class LibvirtStopNBDServerCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + private void resetService(String unitName) { + Script resetScript = new Script("/bin/bash", logger); + resetScript.add("-c"); + resetScript.add(String.format("systemctl reset-failed %s || true", unitName)); + resetScript.execute(); + } + + @Override + public Answer execute(StopNBDServerCommand cmd, LibvirtComputingResource resource) { + try { + String unitName = "qemu-nbd-" + cmd.getTransferId().hashCode(); + + // Check if the service is running + Script checkScript = new Script("/bin/bash", logger); + checkScript.add("-c"); + checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String checkResult = checkScript.execute(); + if (checkResult != null) { + // Service is not running, but still reset-failed to clear any stale state + logger.info(String.format("qemu-nbd service %s is not running, resetting failed state", unitName)); + resetService(unitName); + return new Answer(cmd, true, "Image transfer finalized"); + } + + // Stop the systemd service + Script stopScript = new Script("/bin/bash", logger); + stopScript.add("-c"); + stopScript.add(String.format("systemctl stop %s", unitName)); + stopScript.execute(); + resetService(unitName); + + return new Answer(cmd, true, "Image transfer finalized"); + + } catch (Exception e) { + logger.error("Error finalizing image transfer for upload", e); + return new Answer(cmd, false, "Error finalizing image transfer: " + e.getMessage()); + } + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapperTest.java new file mode 100644 index 000000000000..8782f592aed3 --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapperTest.java @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.kvm.resource.wrapper; + +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.eq; + +import org.apache.cloudstack.backup.CreateImageTransferAnswer; +import org.apache.cloudstack.backup.CreateImageTransferCommand; +import org.apache.cloudstack.backup.ImageTransfer; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.ImageServerControlSocket; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.utils.script.Script; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtCreateImageTransferCommandWrapperTest { + + private LibvirtCreateImageTransferCommandWrapper wrapper; + private CreateImageTransferCommand command; + private LibvirtComputingResource resource; + + @Before + public void setUp() { + wrapper = new LibvirtCreateImageTransferCommandWrapper(); + command = Mockito.mock(CreateImageTransferCommand.class); + resource = Mockito.mock(LibvirtComputingResource.class); + } + + @Test + public void testExecuteBlankTransferIdReturnsFailure() { + Mockito.when(command.getTransferId()).thenReturn(""); + Mockito.when(command.getBackend()).thenReturn(ImageTransfer.Backend.nbd); + + Answer answer = wrapper.execute(command, resource); + + Assert.assertFalse(answer.getResult()); + Assert.assertEquals("transferId is empty.", answer.getDetails()); + } + + @Test + public void testExecuteNbdBackendSuccessReturnsUrl() { + Mockito.when(command.getTransferId()).thenReturn("tr-1"); + Mockito.when(command.getBackend()).thenReturn(ImageTransfer.Backend.nbd); + Mockito.when(command.getIdleTimeoutSeconds()).thenReturn(120); + Mockito.when(command.getSocket()).thenReturn("sock-1"); + Mockito.when(command.getExportName()).thenReturn("vol-1"); + Mockito.when(command.getCheckpointId()).thenReturn("cp-1"); + + Mockito.when(resource.getImageServerPath()).thenReturn("/opt/cloudstack/image/server.py"); + Mockito.when(resource.getImageServerListenAddress()).thenReturn(""); + Mockito.when(resource.getPrivateIp()).thenReturn("10.0.0.10"); + Mockito.when(resource.isImageServerTlsEnabled()).thenReturn(false); + + try (MockedConstruction + + diff --git a/ui/src/components/view/DetailsTab.vue b/ui/src/components/view/DetailsTab.vue index 4145eeb9be6d..ecff34b2949d 100644 --- a/ui/src/components/view/DetailsTab.vue +++ b/ui/src/components/view/DetailsTab.vue @@ -97,7 +97,7 @@ {{ $t(dataResource[item].toLowerCase()) }} {{ dataResource[item] }} -
+
{{ $toLocaleDate(dataResource[item]) }}
{{ dataResource[item] }}
@@ -168,7 +168,7 @@
{{ dataResource[item] }}
- +
{{ $t('label.' + item.replace('date', '.date.and.time'))}}
diff --git a/ui/src/components/view/InfoCard.vue b/ui/src/components/view/InfoCard.vue index 3094bf58dbcc..41c822de238d 100644 --- a/ui/src/components/view/InfoCard.vue +++ b/ui/src/components/view/InfoCard.vue @@ -120,12 +120,18 @@ -
+
{{ $t('label.status') }}
+
+
{{ $t('label.apikeyaccess') }}
+
+ +
+
{{ $t('label.allocationstate') }}
@@ -159,6 +165,42 @@
+
+
+ + + {{ $t('label.apikey') }} + + +
+ {{ resource.apikey.substring(0, 20) }}... +
+

+
+ + + {{ $t('label.secretkey') }} + + +
+ {{ resource.secretkey.substring(0, 20) }}... +
+
+
{{ $t('label.ostypename') }}
@@ -796,6 +838,14 @@ {{ resource.account }}
+
+
{{ $t('label.user') }}
+
+ + {{ resource.username }} + {{ resource.username }} +
+
{{ $t('label.role') }}
@@ -883,54 +933,6 @@ :osCategoryId="osCategoryId" />
- - - - +
@@ -231,10 +243,14 @@ export default { socialLogin: false, googleprovider: false, githubprovider: false, + keycloakprovider: false, googleredirecturi: '', githubredirecturi: '', + keycloakredirecturi: '', googleclientid: '', githubclientid: '', + keycloakclientid: '', + keycloakauthorizeurl: '', loginType: 0, state: { time: 60, @@ -325,8 +341,14 @@ export default { this.githubclientid = item.clientid this.githubredirecturi = item.redirecturi } + if (item.provider === 'keycloak') { + this.keycloakprovider = item.enabled + this.keycloakclientid = item.clientid + this.keycloakredirecturi = item.redirecturi + this.keycloakauthorizeurl = item.authorizeurl + } }) - this.socialLogin = this.googleprovider || this.githubprovider + this.socialLogin = this.googleprovider || this.githubprovider || this.keycloakprovider } }) postAPI('forgotPassword', {}).then(response => { @@ -362,6 +384,10 @@ export default { this.handleDomain() this.$store.commit('SET_OAUTH_PROVIDER_USED_TO_LOGIN', 'google') }, + handleKeycloakProviderAndDomain () { + this.handleDomain() + this.$store.commit('SET_OAUTH_PROVIDER_USED_TO_LOGIN', 'keycloak') + }, handleDomain () { const values = toRaw(this.form) if (!values.domain) { @@ -401,6 +427,20 @@ export default { return `${rootUrl}?${qs.toString()}` }, + getKeycloakUrl (from) { + const rootURl = this.keycloakauthorizeurl + const options = { + redirect_uri: this.keycloakredirecturi, + client_id: this.keycloakclientid, + response_type: 'code', + scope: 'openid email', + state: 'cloudstack' + } + + const qs = new URLSearchParams(options) + + return `${rootURl}?${qs.toString()}` + }, handleSubmit (e) { e.preventDefault() if (this.state.loginBtn) return From 21e4475d961a4cf645ccf043158f6d2bb7655a61 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Tue, 23 Jun 2026 20:11:19 +0530 Subject: [PATCH 28/28] Optimize the DB updates to use bulk UPDATE instead of row-level locks. (#13349) Co-authored-by: Aaron Chung --- .../com/cloud/alert/dao/AlertDaoImpl.java | 28 ++++++------- .../com/cloud/event/dao/EventDaoImpl.java | 29 ++++++++------ .../java/com/cloud/host/dao/HostDaoImpl.java | 22 +++++++---- .../dao/SecurityGroupWorkDaoImpl.java | 39 +++++-------------- 4 files changed, 53 insertions(+), 65 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/alert/dao/AlertDaoImpl.java b/engine/schema/src/main/java/com/cloud/alert/dao/AlertDaoImpl.java index 94d01f472ba5..97b7c54f0844 100644 --- a/engine/schema/src/main/java/com/cloud/alert/dao/AlertDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/alert/dao/AlertDaoImpl.java @@ -20,6 +20,7 @@ import java.util.List; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.alert.AlertVO; @@ -28,7 +29,7 @@ import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; -import com.cloud.utils.db.TransactionLegacy; +import com.cloud.utils.db.UpdateBuilder; @Component public class AlertDaoImpl extends GenericDaoBase implements AlertDao { @@ -107,25 +108,20 @@ public boolean archiveAlert(List ids, String type, Date startDate, Date en } sc.setParameters("archived", false); - boolean result = true; - ; List alerts = listBy(sc); if (ids != null && alerts.size() < ids.size()) { - result = false; - return result; + return false; } - if (alerts != null && !alerts.isEmpty()) { - TransactionLegacy txn = TransactionLegacy.currentTxn(); - txn.start(); - for (AlertVO alert : alerts) { - alert = lockRow(alert.getId(), true); - alert.setArchived(true); - update(alert.getId(), alert); - txn.commit(); - } - txn.close(); + + if (CollectionUtils.isEmpty(alerts)) { + return true; } - return result; + + AlertVO alertForUpdate = createForUpdate(); + alertForUpdate.setArchived(true); + UpdateBuilder ub = getUpdateBuilder(alertForUpdate); + update(ub, sc, null); + return true; } @Override diff --git a/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java b/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java index e748e98900eb..9417ddd12595 100644 --- a/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java @@ -18,8 +18,10 @@ import java.util.Date; import java.util.List; +import java.util.stream.Collectors; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.event.Event.State; @@ -29,12 +31,13 @@ import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; -import com.cloud.utils.db.TransactionLegacy; +import com.cloud.utils.db.UpdateBuilder; @Component public class EventDaoImpl extends GenericDaoBase implements EventDao { protected final SearchBuilder CompletedEventSearch; protected final SearchBuilder ToArchiveOrDeleteEventSearch; + protected final SearchBuilder ArchiveByIdsSearch; public EventDaoImpl() { CompletedEventSearch = createSearchBuilder(); @@ -51,6 +54,10 @@ public EventDaoImpl() { ToArchiveOrDeleteEventSearch.and("createdDateL", ToArchiveOrDeleteEventSearch.entity().getCreateDate(), Op.LTEQ); ToArchiveOrDeleteEventSearch.and("archived", ToArchiveOrDeleteEventSearch.entity().getArchived(), Op.EQ); ToArchiveOrDeleteEventSearch.done(); + + ArchiveByIdsSearch = createSearchBuilder(); + ArchiveByIdsSearch.and("id", ArchiveByIdsSearch.entity().getId(), Op.IN); + ArchiveByIdsSearch.done(); } @Override @@ -100,16 +107,16 @@ public List listToArchiveOrDeleteEvents(List ids, String type, Da @Override public void archiveEvents(List events) { - if (events != null && !events.isEmpty()) { - TransactionLegacy txn = TransactionLegacy.currentTxn(); - txn.start(); - for (EventVO event : events) { - event = lockRow(event.getId(), true); - event.setArchived(true); - update(event.getId(), event); - txn.commit(); - } - txn.close(); + if (CollectionUtils.isEmpty(events)) { + return; } + + List ids = events.stream().map(EventVO::getId).collect(Collectors.toList()); + SearchCriteria sc = ArchiveByIdsSearch.create(); + sc.setParameters("id", ids.toArray(new Object[ids.size()])); + EventVO eventForUpdate = createForUpdate(); + eventForUpdate.setArchived(true); + UpdateBuilder ub = getUpdateBuilder(eventForUpdate); + update(ub, sc, null); } } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index cd4423dfa269..5b8a38b8e5b4 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -646,16 +646,22 @@ private void resetHosts(long managementServerId, long lastPingSecondsAfter) { sc.setParameters("lastPinged", lastPingSecondsAfter); sc.setParameters("status", Status.Disconnected, Status.Down, Status.Alert); - StringBuilder sb = new StringBuilder(); - List hosts = lockRows(sc, null, true); // exclusive lock - for (HostVO host : hosts) { - host.setManagementServerId(null); - update(host.getId(), host); - sb.append(host.getId()); - sb.append(" "); + // SELECT before bulk UPDATE to preserve per-host-ID trace logging — the bulk UPDATE + // cannot return which rows it matched since the WHERE column is being set to NULL + if (logger.isTraceEnabled()) { + List hosts = listBy(sc); + StringBuilder sb = new StringBuilder(); + for (HostVO host : hosts) { + sb.append(host.getId()); + sb.append(" "); + } + logger.trace("Following hosts will be reset: {}", sb); } - logger.trace("Following hosts got reset: {}", sb); + HostVO host = createForUpdate(); + host.setManagementServerId(null); + UpdateBuilder ub = getUpdateBuilder(host); + update(ub, sc, null); } /* diff --git a/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupWorkDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupWorkDaoImpl.java index 327d12c759a7..3180ef30a3ce 100644 --- a/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupWorkDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupWorkDaoImpl.java @@ -116,7 +116,7 @@ public SecurityGroupWorkVO take(long serverId) { //ensure that there is no job in Processing state for the same VM processing = true; if (logger.isTraceEnabled()) { - logger.trace("Security Group work take: found a job in Scheduled and Processing vmid=" + work.getInstanceId()); + logger.trace("Security Group work take: found a job in Scheduled and Processing vmid={}", work.getInstanceId()); } } work.setServerId(serverId); @@ -141,26 +141,16 @@ public SecurityGroupWorkVO take(long serverId) { } @Override - @DB public void updateStep(Long vmId, Long logSequenceNumber, Step step) { - final TransactionLegacy txn = TransactionLegacy.currentTxn(); - txn.start(); SearchCriteria sc = VmIdSeqNumSearch.create(); sc.setParameters("vmId", vmId); sc.setParameters("seqno", logSequenceNumber); - final Filter filter = new Filter(SecurityGroupWorkVO.class, null, true, 0l, 1l); - - final List vos = lockRows(sc, filter, true); - if (vos.size() == 0) { - txn.commit(); - return; - } - SecurityGroupWorkVO work = vos.get(0); - work.setStep(step); - update(work.getId(), work); - - txn.commit(); + SecurityGroupWorkVO workForUpdate = createForUpdate(); + workForUpdate.setStep(step); + // LIMIT 1 preserves the original single-row semantics: op_nwgrp_work has no + // uniqueness on (instance_id, seq_no), so without it duplicate rows would all be updated. + update(workForUpdate, sc, 1); } @Override @@ -172,21 +162,10 @@ public SecurityGroupWorkVO findByVmIdStep(long vmId, Step step) { } @Override - @DB public void updateStep(Long workId, Step step) { - final TransactionLegacy txn = TransactionLegacy.currentTxn(); - txn.start(); - - SecurityGroupWorkVO work = lockRow(workId, true); - if (work == null) { - txn.commit(); - return; - } - work.setStep(step); - update(work.getId(), work); - - txn.commit(); - + SecurityGroupWorkVO workForUpdate = createForUpdate(); + workForUpdate.setStep(step); + update(workId, workForUpdate); } @Override