diff --git a/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java index e1affb311ec6..e4df538fc189 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java +++ b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java @@ -31,6 +31,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import com.cloud.storage.StoragePool; import com.cloud.utils.component.PluggableService; /** @@ -103,4 +104,10 @@ public interface KVMBackupExportService extends Configurable, PluggableService { * List Compatible Data Center Ids for the service */ List listCompatibleDataCenterIds(); + + /** + * Whether the storage driver of the pool exports the pool's volumes for backups itself (block storage), + * instead of the hypervisor reading the volume files + */ + boolean isStorageDriverBackupExportSupported(StoragePool pool); } diff --git a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java index 95b56c9a9c38..e6baa15f0be1 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java @@ -17,7 +17,10 @@ package org.apache.cloudstack.backup; +import java.util.Map; + import com.cloud.agent.api.Command; +import com.cloud.storage.Storage; public class CreateImageTransferCommand extends Command { private String transferId; @@ -28,6 +31,9 @@ public class CreateImageTransferCommand extends Command { private String file; private ImageTransfer.Backend backend; private int idleTimeoutSeconds; + private Storage.StoragePoolType extentsPoolType; + private String extentsPoolUuid; + private Map extentsSource; public CreateImageTransferCommand() { } @@ -91,4 +97,27 @@ public String getCheckpointId() { public int getIdleTimeoutSeconds() { return idleTimeoutSeconds; } + + /** + * Has the pool's storage list the extents of the export, the ones that changed since the last backup and the + * ones that read as zeros, instead of a dirty bitmap (see the checkpoint id) and the allocation of the export. + * @param source driver specific parameters, see {@code KVMStoragePool.writeExtents} + */ + public void setExtentsSource(Storage.StoragePoolType poolType, String poolUuid, Map source) { + this.extentsPoolType = poolType; + this.extentsPoolUuid = poolUuid; + this.extentsSource = source; + } + + public Storage.StoragePoolType getExtentsPoolType() { + return extentsPoolType; + } + + public String getExtentsPoolUuid() { + return extentsPoolUuid; + } + + public Map getExtentsSource() { + return extentsSource; + } } diff --git a/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java index 84d9b1ff8186..0b089e4273a4 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java @@ -18,9 +18,14 @@ package org.apache.cloudstack.backup; import com.cloud.agent.api.Command; +import com.cloud.storage.Storage; public class FinalizeImageTransferCommand extends Command { private String transferId; + private String convertSource; + private Storage.StoragePoolType convertPoolType; + private String convertPoolUuid; + private String convertVolumePath; public FinalizeImageTransferCommand() { } @@ -33,6 +38,34 @@ public String getTransferId() { return transferId; } + /** + * Has the host convert the qcow2 image uploaded to a temporary device into the volume, see + * {@code PrimaryDataStoreDriver.createUploadStaging}. + * @param source the temporary device + */ + public void setConvertInto(String source, Storage.StoragePoolType poolType, String poolUuid, String volumePath) { + this.convertSource = source; + this.convertPoolType = poolType; + this.convertPoolUuid = poolUuid; + this.convertVolumePath = volumePath; + } + + public String getConvertSource() { + return convertSource; + } + + public Storage.StoragePoolType getConvertPoolType() { + return convertPoolType; + } + + public String getConvertPoolUuid() { + return convertPoolUuid; + } + + public String getConvertVolumePath() { + return convertVolumePath; + } + @Override public boolean executeInSequence() { return true; diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java index 07e50f11fd37..7a87ddf1458f 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java @@ -29,6 +29,7 @@ public class StartNBDServerCommand extends Command { private String fromCheckpointId; @LogLevel(LogLevel.Log4jLevel.Off) private byte[] passphrase; + private Long blockDeviceSizeBytes; public StartNBDServerCommand() { } @@ -43,6 +44,14 @@ protected StartNBDServerCommand(String transferId, String exportName, String vol this.passphrase = passphrase; } + /** + * Exports a raw block device. Only its first sizeBytes are exported, as the device can be larger than the volume. + */ + protected StartNBDServerCommand(String transferId, String exportName, String devicePath, long sizeBytes, String socket, String direction) { + this(transferId, exportName, devicePath, socket, direction, null, null); + this.blockDeviceSizeBytes = sizeBytes; + } + public String getExportName() { return exportName; } @@ -75,4 +84,11 @@ public String getFromCheckpointId() { public byte[] getPassphrase() { return passphrase; } + + /** + * Size to export if the volume path is a raw block device, null if it is an image file. + */ + public Long getBlockDeviceSizeBytes() { + return blockDeviceSizeBytes; + } } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java index 37e2d2dd4b12..28e84d0e99ae 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java @@ -249,4 +249,63 @@ default List listVolumesForImport(StoragePool pool, String vo */ default void volumeManagementChanged(StoragePool pool, Volume volume, boolean imported) { } + + /** + * Whether the driver serves the volumes of the pool for image based backups (KVM backup export) from + * its own storage snapshots, see {@link #startBackupExport}, instead of the hypervisor reading the volume files. + */ + default boolean canExportVolumesForBackup(StoragePool pool) { + return false; + } + + /** + * Takes one crash consistent storage snapshot of the volumes and makes each volume of it readable as a raw + * device on a host. {@link #finishBackupExport} has to be called afterwards in any case, also if this fails. + * + * @param exportName name of the export, unique per backup + * @param volumes volumes of one VM, all on this pool + * @param changeTrackingIds per volume id, the change tracking id of the last successful backup of the volume + * (see {@link VolumeBackupExport#getChangeTrackingId()}), missing if there is none + * @return per volume where to read it + */ + default List startBackupExport(StoragePool pool, String exportName, List volumes, + Map changeTrackingIds) { + throw new UnsupportedOperationException("The storage driver does not export volumes for backups"); + } + + /** + * Releases everything {@link #startBackupExport} set up for the export (devices, snapshot). Does nothing for + * parts that are already released, so it can be called again after a failure. + */ + default void finishBackupExport(StoragePool pool, String exportName, List volumes) { + throw new UnsupportedOperationException("The storage driver does not export volumes for backups"); + } + + /** + * Makes the volume writable as a raw device on a host, to restore a backup into it. + */ + default VolumeBackupExport prepareVolumeForRestore(StoragePool pool, VolumeInfo volume) { + throw new UnsupportedOperationException("The storage driver does not export volumes for backups"); + } + + /** + * Creates a temporary raw device on a host, to upload an image of the volume in another format than raw + * (qcow2) into. The host converts the image into the device of {@link #prepareVolumeForRestore} afterwards. + * Without, only raw images can be uploaded into the volumes of the pool. + * + * @param name name of the device, unique per upload + * @param hostId host to create the device on + * @param sizeBytes the most the image can take + * @return where to write the image, null if the driver has no temporary devices + */ + default VolumeBackupExport createUploadStaging(StoragePool pool, VolumeInfo volume, String name, long hostId, + long sizeBytes) { + return null; + } + + /** + * Deletes the device of {@link #createUploadStaging}. Does nothing if there is none. + */ + default void deleteUploadStaging(StoragePool pool, String name) { + } } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeBackupExport.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeBackupExport.java new file mode 100644 index 000000000000..2f1c126611bc --- /dev/null +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeBackupExport.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.engine.subsystem.api.storage; + +import java.util.Map; + +/** + * Where a storage driver made a volume accessible as a raw device for an image based backup + * (read only, see {@link PrimaryDataStoreDriver#startBackupExport}) or a restore into it + * (see {@link PrimaryDataStoreDriver#prepareVolumeForRestore}). + */ +public class VolumeBackupExport { + private long volumeId; + private long hostId; + private String devicePath; + private long sizeBytes; + private boolean incremental; + private String changeTrackingId; + private Map extentsSource; + + public VolumeBackupExport() { + } + + public VolumeBackupExport(long volumeId, long hostId, String devicePath, long sizeBytes) { + this.volumeId = volumeId; + this.hostId = hostId; + this.devicePath = devicePath; + this.sizeBytes = sizeBytes; + } + + public long getVolumeId() { + return volumeId; + } + + /** + * The host the device is accessible on. + */ + public long getHostId() { + return hostId; + } + + public String getDevicePath() { + return devicePath; + } + + /** + * Size of the volume in bytes. The device can be larger, only this much of it is the volume's data. + */ + public long getSizeBytes() { + return sizeBytes; + } + + /** + * True if only the extents {@link #getExtentsSource()} lists as dirty changed since the backup with the + * change tracking id given to {@link PrimaryDataStoreDriver#startBackupExport}, false if the whole volume + * has to be read. + */ + public boolean isIncremental() { + return incremental; + } + + public void setIncremental(boolean incremental) { + this.incremental = incremental; + } + + /** + * The storage's change tracking id of this backup, to pass to the next {@link PrimaryDataStoreDriver#startBackupExport} + * once this backup succeeded. Null if the storage does not track changes for the volume. + */ + public String getChangeTrackingId() { + return changeTrackingId; + } + + public void setChangeTrackingId(String changeTrackingId) { + this.changeTrackingId = changeTrackingId; + } + + /** + * Driver specific parameters the host's storage adaptor uses to list the extents of the export: the ones that + * changed (of an incremental export) and the ones that read as zeros. Null if the storage lists none, then the + * whole volume counts as changed data. + */ + public Map getExtentsSource() { + return extentsSource; + } + + public void setExtentsSource(Map extentsSource) { + this.extentsSource = extentsSource; + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java index ec9b927b63e4..450272b9ca8d 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java @@ -198,6 +198,10 @@ public Backend getBackend() { return backend; } + public String getFile() { + return file; + } + @Override public String getSignedTicketId() { return signedTicketId; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java index 7cf05da9b211..481e54a09f82 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java @@ -18,6 +18,9 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; @@ -25,15 +28,18 @@ import org.apache.cloudstack.backup.CreateImageTransferCommand; import org.apache.cloudstack.backup.ImageTransfer; import org.apache.cloudstack.storage.resource.IpTablesHelper; +import org.apache.commons.collections.MapUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import com.cloud.agent.api.Answer; import com.cloud.hypervisor.kvm.resource.ImageServerControlSocket; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.utils.StringUtils; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; @ResourceWrapper(handles = CreateImageTransferCommand.class) @@ -42,6 +48,23 @@ public class LibvirtCreateImageTransferCommandWrapper extends CommandWrapper { + /** what qemu-img info calls the external data file of a qcow2 image */ + private static final String QEMU_IMG_INFO_DATA_FILE = "data_file"; + protected Logger logger = LogManager.getLogger(getClass()); private void resetService(String unitName) { Script resetScript = new Script("/bin/bash", logger); @@ -86,16 +98,55 @@ public Answer execute(FinalizeImageTransferCommand cmd, LibvirtComputingResource } int activeTransfers = ImageServerControlSocket.unregisterTransfer(transferId); + LibvirtCreateImageTransferCommandWrapper.deleteExtentsFile(transferId); + String details = "Image transfer finalized."; if (activeTransfers < 0) { logger.warn("Could not reach image server to unregister transfer {}; assuming server is down", transferId); stopImageServer(imageServerPort, resource); - return new Answer(cmd, true, "Image transfer finalized (server unreachable, forced stop)."); + details = "Image transfer finalized (server unreachable, forced stop)."; + } else if (activeTransfers == 0) { + stopImageServer(imageServerPort, resource); } - if (activeTransfers == 0) { - stopImageServer(imageServerPort, resource); + if (cmd.getConvertSource() != null) { + // the image server has let go of the uploaded image by now + String error = convertUploadedImage(cmd, resource); + if (error != null) { + return new Answer(cmd, false, error); + } } + return new Answer(cmd, true, details); + } + + /** + * Converts the qcow2 image uploaded to a temporary device into the volume, see + * {@code PrimaryDataStoreDriver.createUploadStaging}. + * @return null if done, else the error + */ + protected String convertUploadedImage(FinalizeImageTransferCommand cmd, LibvirtComputingResource resource) { + final String source = cmd.getConvertSource(); + try { + KVMStoragePool pool = resource.getStoragePoolMgr().getStoragePool(cmd.getConvertPoolType(), cmd.getConvertPoolUuid()); + KVMPhysicalDisk volume = pool.getPhysicalDisk(cmd.getConvertVolumePath()); + long timeout = cmd.getWait() > 0 ? cmd.getWait() * 1000L : resource.getCmdsTimeout(); + QemuImg qemu = createQemuImg(timeout, pool.isVolumeZeroInitialized(cmd.getConvertVolumePath())); + QemuImgFile image = new QemuImgFile(source, QemuImg.PhysicalDiskFormat.QCOW2); + // the image comes from outside, it must not pull other files of the host into the volume + Map info = qemu.info(image); + if (!"qcow2".equals(info.get(QemuImg.FILE_FORMAT)) || info.containsKey(QemuImg.BACKING_FILE) + || info.containsKey(QEMU_IMG_INFO_DATA_FILE)) { + return String.format("The image uploaded to %s is no qcow2 image without backing and data file", source); + } + qemu.convert(image, new QemuImgFile(volume.getPath(), QemuImg.PhysicalDiskFormat.RAW), true); + logger.info("Converted the image uploaded to {} into {}", source, volume.getPath()); + return null; + } catch (QemuImgException | LibvirtException | CloudRuntimeException e) { + logger.error("Failed to convert the image uploaded to {} into volume {}", source, cmd.getConvertVolumePath(), e); + return "Failed to convert the uploaded image into the volume: " + e.getMessage(); + } + } - return new Answer(cmd, true, "Image transfer finalized."); + protected QemuImg createQemuImg(long timeout, boolean targetIsZero) throws LibvirtException { + return new QemuImg(timeout, targetIsZero, true); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java index 661344677640..023bfc3fcb60 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java @@ -52,7 +52,11 @@ && isBitmapPresentOnDisk(volumePath, cmd.getFromCheckpointId())) { String readOnlyArg = cmd.getDirection().equals("download") ? "--read-only" : ""; String imageArg = volumePath; String secretArg = ""; - if (passphrase != null && passphrase.length > 0) { + if (cmd.getBlockDeviceSizeBytes() != null) { + // the device can be larger than the volume (e.g. DRBD), so only export the volume's size of it + imageArg = String.format("--image-opts driver=raw,size=%d,file.driver=host_device,file.filename=%s", + cmd.getBlockDeviceSizeBytes(), volumePath); + } else if (passphrase != null && passphrase.length > 0) { KeyFile srcKey = new KeyFile(passphrase); secretArg = String.format("--object secret,id=sec0,file=%s", srcKey); imageArg = String.format("--image-opts driver=qcow2,file.driver=file,file.filename=%s,encrypt.key-secret=sec0", volumePath); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java index 58665fdb5766..13d02aff24c7 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java @@ -202,4 +202,18 @@ default void deleteBackupScratch(String name) { throw new CloudRuntimeException(String.format("Storage pool %s of type %s cannot provide backup scratch space", getUuid(), getType())); } + + /** + * Writes the extents of a volume's backup export, as the storage lists them, to {@code file}: a JSON array of + * extents like the ones of the ovirt-imageio extents API, objects with {@code start} and {@code length} in bytes + * and the flags {@code dirty} (changed since the last backup) and {@code zero} (reads as zeros). An extent + * without the dirty flag counts as changed, one without the zero flag as data. The image server reports them + * as the extents of the transfer. + * + * @param source the storage driver's parameters naming the volume and its backup + */ + default void writeExtents(Map source, String file) { + throw new CloudRuntimeException(String.format("Storage pool %s of type %s does not list the extents of its volumes", + getUuid(), getType())); + } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapperTest.java index 8782f592aed3..8d04f45a4cb8 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapperTest.java @@ -17,8 +17,11 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import java.util.Map; + import org.apache.cloudstack.backup.CreateImageTransferAnswer; import org.apache.cloudstack.backup.CreateImageTransferCommand; import org.apache.cloudstack.backup.ImageTransfer; @@ -34,6 +37,10 @@ import com.cloud.agent.api.Answer; import com.cloud.hypervisor.kvm.resource.ImageServerControlSocket; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.storage.Storage; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; @RunWith(MockitoJUnitRunner.class) @@ -118,4 +125,61 @@ public void testExecuteRegisterTransferFailureReturnsFailure() { Assert.assertEquals("Failed to register transfer with image server.", answer.getDetails()); } } + + private KVMStoragePool mockNbdTransferWithExtentsSource(String transferId, Map source) { + Mockito.when(command.getTransferId()).thenReturn(transferId); + Mockito.when(command.getBackend()).thenReturn(ImageTransfer.Backend.nbd); + Mockito.when(command.getSocket()).thenReturn("sock-3"); + Mockito.when(command.getExportName()).thenReturn("vol-3"); + Mockito.when(command.getExtentsSource()).thenReturn(source); + Mockito.when(command.getExtentsPoolType()).thenReturn(Storage.StoragePoolType.Linstor); + Mockito.when(command.getExtentsPoolUuid()).thenReturn("pool-1"); + + Mockito.when(resource.getImageServerPath()).thenReturn("/opt/cloudstack/image/server.py"); + Mockito.when(resource.getImageServerListenAddress()).thenReturn("192.168.10.10"); + KVMStoragePoolManager poolManager = Mockito.mock(KVMStoragePoolManager.class); + KVMStoragePool pool = Mockito.mock(KVMStoragePool.class); + Mockito.when(resource.getStoragePoolMgr()).thenReturn(poolManager); + Mockito.when(poolManager.getStoragePool(Storage.StoragePoolType.Linstor, "pool-1")).thenReturn(pool); + return pool; + } + + @Test + public void testExecuteWithExtentsSourceHasImageServerReadTheListedExtents() { + Map source = Map.of("snapshot", "ckp-1"); + KVMStoragePool pool = mockNbdTransferWithExtentsSource("tr-3", source); + + try (MockedConstruction