mirror of
https://github.com/binaricat/Netcatty.git
synced 2026-10-09 15:58:34 +00:00
fix(sftp): bound multi-select downloads and preserve dest mtime
Address Codex review on PR #2772: serialize local blob reads, bound multi-select root downloads before pre-scan, replace remaining Unicode em dashes, and stamp destination mtimes after successful transfers so skip-unchanged can match on a later pass. Co-authored-by: 陈大猫 <binaricat@users.noreply.github.com>
This commit is contained in:
@@ -289,7 +289,7 @@ export async function transferDiscoveredFiles(
|
||||
? {
|
||||
...t,
|
||||
status: "failed" as TransferStatus,
|
||||
error: "SFTP session lost — reconnect and resume remaining files",
|
||||
error: "SFTP session lost - reconnect and resume remaining files",
|
||||
speed: 0,
|
||||
endTime: Date.now(),
|
||||
}
|
||||
|
||||
@@ -1048,7 +1048,7 @@ export function useSftpDirectoryTransferOps({
|
||||
? {
|
||||
...t,
|
||||
status: "failed" as TransferStatus,
|
||||
error: "SFTP session lost — reconnect and resume remaining files",
|
||||
error: "SFTP session lost - reconnect and resume remaining files",
|
||||
speed: 0,
|
||||
endTime: Date.now(),
|
||||
}
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
import { useCallback, useRef, useState } from "react";
|
||||
import type { SftpFileEntry } from "../../../types";
|
||||
import type { TransferStatus } from "../../../domain/models";
|
||||
import { getParentPath, joinPath as joinFsPath } from "../../../application/state/sftp/utils";
|
||||
import {
|
||||
DEFAULT_SFTP_FILE_TRANSFER_CONCURRENCY,
|
||||
runBoundedConcurrency,
|
||||
} from "../../../application/state/sftp/transferConcurrency";
|
||||
import { logger } from "../../../lib/logger";
|
||||
import { toast } from "../../ui/toast";
|
||||
import { getFileExtension, getLanguageId, FileOpenerType, SystemAppInfo } from "../../../lib/sftpFileUtils";
|
||||
@@ -12,6 +17,14 @@ import type { TextEditorModalSnapshot } from "../../TextEditorModal";
|
||||
import type { UseSftpViewFileOpsParams, UseSftpViewFileOpsResult } from "./useSftpViewFileOps.types";
|
||||
import { assertSftpFileFitsBuiltinEditor } from "../sftpEditorFileLimits";
|
||||
|
||||
/** Local multi-select blob downloads read whole files into ArrayBuffers. */
|
||||
const LOCAL_BLOB_DOWNLOAD_CONCURRENCY = 1;
|
||||
/**
|
||||
* Multi-select roots each run folder pre-scan before the transfer scheduler.
|
||||
* Bound them so listing fan-out cannot multiply by selection size.
|
||||
*/
|
||||
const MULTI_SELECT_ROOT_DOWNLOAD_CONCURRENCY = DEFAULT_SFTP_FILE_TRANSFER_CONCURRENCY;
|
||||
|
||||
export const useSftpViewFileOps = ({
|
||||
sftpRef,
|
||||
behaviorRef,
|
||||
@@ -538,7 +551,14 @@ export const useSftpViewFileOps = ({
|
||||
if (!pane.connection) return;
|
||||
|
||||
if (pane.connection.isLocal) {
|
||||
await Promise.allSettled(files.map((file) => handleDownloadFileForSide(side, file)));
|
||||
// Sequential: each local download materializes a full ArrayBuffer.
|
||||
await runBoundedConcurrency(
|
||||
files,
|
||||
LOCAL_BLOB_DOWNLOAD_CONCURRENCY,
|
||||
async (file) => {
|
||||
await handleDownloadFileForSide(side, file);
|
||||
},
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -556,31 +576,40 @@ export const useSftpViewFileOps = ({
|
||||
const selectedDirectory = await selectDirectory(t("sftp.context.download"));
|
||||
if (!selectedDirectory) return;
|
||||
|
||||
// Enqueue in parallel — each downloadToLocal uses the dedicated transfer
|
||||
// pool / host scheduler. Awaiting one-by-one forced serial multi-select
|
||||
// throughput even though per-file SFTP I/O is already pipelined.
|
||||
const results = await Promise.allSettled(files.map(async (file) => {
|
||||
const sourcePath = sftpRef.current.joinPath(pane.connection.currentPath, file.name);
|
||||
const targetPath = joinFsPath(selectedDirectory, file.name);
|
||||
const isDirectory = isNavigableDirectory(file);
|
||||
const fileSize = typeof file.size === "string" ? parseInt(file.size, 10) || 0 : (file.size || 0);
|
||||
// Bound root jobs: directory roots run full pre-scan before the scheduler,
|
||||
// so unbounded Promise.allSettled would multiply LIST concurrency.
|
||||
const results: Array<PromiseSettledResult<{ file: SftpFileEntry; status: TransferStatus }>> = [];
|
||||
await runBoundedConcurrency(
|
||||
files,
|
||||
MULTI_SELECT_ROOT_DOWNLOAD_CONCURRENCY,
|
||||
async (file, index) => {
|
||||
try {
|
||||
const sourcePath = sftpRef.current.joinPath(pane.connection.currentPath, file.name);
|
||||
const targetPath = joinFsPath(selectedDirectory, file.name);
|
||||
const isDirectory = isNavigableDirectory(file);
|
||||
const fileSize = typeof file.size === "string" ? parseInt(file.size, 10) || 0 : (file.size || 0);
|
||||
|
||||
const status = await sftpRef.current.downloadToLocal({
|
||||
fileName: file.name,
|
||||
sourcePath,
|
||||
targetPath,
|
||||
sftpId,
|
||||
connectionId: pane.connection.id,
|
||||
sourceHostId: pane.connection.hostId,
|
||||
sourceHostLabel: pane.connection.hostLabel,
|
||||
sourceEncoding: pane.filenameEncoding,
|
||||
isDirectory,
|
||||
totalBytes: isDirectory ? undefined : fileSize,
|
||||
});
|
||||
return { file, status };
|
||||
}));
|
||||
const status = await sftpRef.current.downloadToLocal({
|
||||
fileName: file.name,
|
||||
sourcePath,
|
||||
targetPath,
|
||||
sftpId,
|
||||
connectionId: pane.connection.id,
|
||||
sourceHostId: pane.connection.hostId,
|
||||
sourceHostLabel: pane.connection.hostLabel,
|
||||
sourceEncoding: pane.filenameEncoding,
|
||||
isDirectory,
|
||||
totalBytes: isDirectory ? undefined : fileSize,
|
||||
});
|
||||
results[index] = { status: "fulfilled", value: { file, status } };
|
||||
} catch (reason) {
|
||||
results[index] = { status: "rejected", reason };
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
for (const result of results) {
|
||||
if (!result) continue;
|
||||
if (result.status === "fulfilled") {
|
||||
const { file, status } = result.value;
|
||||
if (status === "completed") {
|
||||
|
||||
@@ -862,6 +862,63 @@ async function promoteLocalTransfer(stagedPath, targetPath, options = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the source mtime to the committed destination so skip-unchanged
|
||||
* (size + mtime) can match on a later folder transfer. Best-effort: never
|
||||
* fails the transfer if utimes/setstat is unsupported.
|
||||
*/
|
||||
async function preserveTransferredDestinationMtime(transfer) {
|
||||
try {
|
||||
let mtimeMs = Number(transfer?.sourceSoftIdentity?.mtimeMs);
|
||||
if ((!Number.isFinite(mtimeMs) || mtimeMs <= 0)
|
||||
&& typeof transfer?.captureSourceSoftIdentity === "function") {
|
||||
await transfer.captureSourceSoftIdentity();
|
||||
mtimeMs = Number(transfer?.sourceSoftIdentity?.mtimeMs);
|
||||
}
|
||||
if (!Number.isFinite(mtimeMs) || mtimeMs <= 0) return;
|
||||
|
||||
// Compare / skip logic uses whole seconds; stamp the destination the same way.
|
||||
const mtimeSec = Math.floor(mtimeMs / 1000);
|
||||
if (mtimeSec <= 0) return;
|
||||
const when = new Date(mtimeSec * 1000);
|
||||
|
||||
if (transfer.targetType === "local" && transfer.targetPath) {
|
||||
await fs.promises.utimes(transfer.targetPath, when, when);
|
||||
return;
|
||||
}
|
||||
|
||||
if (transfer.targetType !== "sftp" || !transfer.targetSftpId || !transfer.targetPath) {
|
||||
return;
|
||||
}
|
||||
const client = sftpClients.get(transfer.targetSftpId);
|
||||
// SCP has no SETSTAT; same-host `cp -a` already preserves mtime.
|
||||
if (!client || isScpModeClient(client)) return;
|
||||
|
||||
await requireSftpChannel(client);
|
||||
const encoded = encodePathForSession(
|
||||
transfer.targetSftpId,
|
||||
transfer.targetPath,
|
||||
transfer.targetEncoding,
|
||||
);
|
||||
if (typeof client.setStat === "function") {
|
||||
await client.setStat(encoded, { mtime: mtimeSec, atime: mtimeSec });
|
||||
return;
|
||||
}
|
||||
const sftp = client.sftp;
|
||||
if (!sftp || typeof sftp.setstat !== "function") return;
|
||||
await new Promise((resolve, reject) => {
|
||||
sftp.setstat(encoded, { mtime: mtimeSec, atime: mtimeSec }, (err) => (
|
||||
err ? reject(err) : resolve()
|
||||
));
|
||||
});
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
"[transferBridge] failed to preserve destination mtime:",
|
||||
err?.message || String(err),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function inspectLocalPromotionTarget(targetPath) {
|
||||
const absoluteTargetPath = path.resolve(targetPath);
|
||||
const visitedLinkStates = new Set();
|
||||
@@ -5604,6 +5661,8 @@ async function startTransferNow(event, payload, onProgress) {
|
||||
}
|
||||
|
||||
sendProgress(fileSize, fileSize);
|
||||
// Stamp destination mtime from the source so skip-unchanged can match later.
|
||||
await preserveTransferredDestinationMtime(transfer);
|
||||
logTransferDiag(transfer, "done", {
|
||||
transferred: fileSize,
|
||||
total: fileSize,
|
||||
@@ -6585,6 +6644,7 @@ module.exports = {
|
||||
releaseSftpTransferSession,
|
||||
listTransferSftpIds,
|
||||
_promoteLocalTransferForTests: promoteLocalTransfer,
|
||||
_preserveTransferredDestinationMtimeForTests: preserveTransferredDestinationMtime,
|
||||
_stableLocalFileIdentityForTests: stableLocalFileIdentity,
|
||||
_getWorkerTransferLifecycleEpochCountForTests: () => workerTransferLifecycleEpochs.size,
|
||||
_setWorkerTransferLifecycleEpochForTests: (transferId, epoch) => {
|
||||
|
||||
@@ -10008,3 +10008,50 @@ test("local promotion succeeds when backup still matches validated identity", as
|
||||
transferBridge._stableLocalFileIdentityForTests(originalStat),
|
||||
);
|
||||
});
|
||||
|
||||
test("preserveTransferredDestinationMtime stamps local targets from sourceSoftIdentity", async (t) => {
|
||||
const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "netcatty-preserve-mtime-"));
|
||||
t.after(async () => fs.promises.rm(tempDir, { recursive: true, force: true }));
|
||||
|
||||
const targetPath = path.join(tempDir, "copied.bin");
|
||||
await fs.promises.writeFile(targetPath, Buffer.from("payload"));
|
||||
const before = await fs.promises.stat(targetPath);
|
||||
const sourceMtimeMs = 1_700_000_000_000; // 2023-11-14T22:13:20.000Z
|
||||
|
||||
await transferBridge._preserveTransferredDestinationMtimeForTests({
|
||||
targetType: "local",
|
||||
targetPath,
|
||||
sourceSoftIdentity: { size: 7, mtimeMs: sourceMtimeMs },
|
||||
});
|
||||
|
||||
const after = await fs.promises.stat(targetPath);
|
||||
assert.equal(Math.floor(after.mtimeMs / 1000), Math.floor(sourceMtimeMs / 1000));
|
||||
assert.notEqual(Math.floor(after.mtimeMs / 1000), Math.floor(before.mtimeMs / 1000));
|
||||
});
|
||||
|
||||
test("local-to-local transfer preserves source mtime on the destination", async (t) => {
|
||||
const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "netcatty-local-mtime-"));
|
||||
t.after(async () => fs.promises.rm(tempDir, { recursive: true, force: true }));
|
||||
|
||||
const sourcePath = path.join(tempDir, "source.bin");
|
||||
const targetPath = path.join(tempDir, "target.bin");
|
||||
await fs.promises.writeFile(sourcePath, Buffer.from("hello-mtime"));
|
||||
const sourceMtimeMs = 1_600_000_000_000; // 2020-09-13T12:26:40.000Z
|
||||
await fs.promises.utimes(sourcePath, new Date(sourceMtimeMs), new Date(sourceMtimeMs));
|
||||
|
||||
transferBridge.init({ sftpClients: new Map() });
|
||||
const result = await transferBridge.startTransfer({ sender: createSender() }, {
|
||||
transferId: "local-mtime-preserve",
|
||||
sourcePath,
|
||||
targetPath,
|
||||
sourceType: "local",
|
||||
targetType: "local",
|
||||
totalBytes: 11,
|
||||
resumable: false,
|
||||
});
|
||||
|
||||
assert.equal(result.error, undefined);
|
||||
const targetStat = await fs.promises.stat(targetPath);
|
||||
assert.equal(Math.floor(targetStat.mtimeMs / 1000), Math.floor(sourceMtimeMs / 1000));
|
||||
assert.deepEqual(await fs.promises.readFile(targetPath), Buffer.from("hello-mtime"));
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user