From b9bfd04f9e6f78ba1c9fed05319ff2e282615b3a Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Sun, 22 Feb 2026 19:57:45 +0000 Subject: [PATCH 1/3] Use tar-fs for container copy archives --- package-lock.json | 21 --- packages/testcontainers/package.json | 2 - .../create-archive-to-copy-to-container.ts | 142 ++++++++++++++++++ .../generic-container.test.ts | 47 +++++- .../generic-container/generic-container.ts | 32 +--- .../started-generic-container.ts | 27 +--- 6 files changed, 195 insertions(+), 76 deletions(-) create mode 100644 packages/testcontainers/src/generic-container/create-archive-to-copy-to-container.ts diff --git a/package-lock.json b/package-lock.json index f197dff57..19d3a08cb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7339,16 +7339,6 @@ "@types/node": "*" } }, - "node_modules/@types/archiver": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/@types/archiver/-/archiver-7.0.0.tgz", - "integrity": "sha512-/3vwGwx9n+mCQdYZ2IKGGHEFL30I96UgBlk8EtRDDFQ9uxM1l4O5Ci6r00EMAkiDaTqD9DQ6nVrWRICnBPtzzg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/readdir-glob": "*" - } - }, "node_modules/@types/async-lock": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/@types/async-lock/-/async-lock-1.4.2.tgz", @@ -7687,15 +7677,6 @@ "@types/node": "*" } }, - "node_modules/@types/readdir-glob": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@types/readdir-glob/-/readdir-glob-1.1.5.tgz", - "integrity": "sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/request": { "version": "2.48.13", "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.13.tgz", @@ -19609,7 +19590,6 @@ "dependencies": { "@balena/dockerignore": "^1.0.2", "@types/dockerode": "^4.0.1", - "archiver": "^7.0.1", "async-lock": "^1.4.1", "byline": "^5.0.0", "debug": "^4.4.3", @@ -19624,7 +19604,6 @@ "undici": "^7.22.0" }, "devDependencies": { - "@types/archiver": "^7.0.0", "@types/async-lock": "^1.4.2", "@types/byline": "^4.2.36", "@types/debug": "^4.1.12", diff --git a/packages/testcontainers/package.json b/packages/testcontainers/package.json index 7da7e063d..e26055cb9 100644 --- a/packages/testcontainers/package.json +++ b/packages/testcontainers/package.json @@ -32,7 +32,6 @@ "dependencies": { "@balena/dockerignore": "^1.0.2", "@types/dockerode": "^4.0.1", - "archiver": "^7.0.1", "async-lock": "^1.4.1", "byline": "^5.0.0", "debug": "^4.4.3", @@ -47,7 +46,6 @@ "undici": "^7.22.0" }, "devDependencies": { - "@types/archiver": "^7.0.0", "@types/async-lock": "^1.4.2", "@types/byline": "^4.2.36", "@types/debug": "^4.1.12", diff --git a/packages/testcontainers/src/generic-container/create-archive-to-copy-to-container.ts b/packages/testcontainers/src/generic-container/create-archive-to-copy-to-container.ts new file mode 100644 index 000000000..e54692f3e --- /dev/null +++ b/packages/testcontainers/src/generic-container/create-archive-to-copy-to-container.ts @@ -0,0 +1,142 @@ +import { createWriteStream, promises as fs } from "fs"; +import path from "path"; +import { Readable } from "stream"; +import { pipeline } from "stream/promises"; +import tar from "tar-fs"; +import tmp from "tmp"; +import { Content, ContentToCopy, DirectoryToCopy, FileToCopy } from "../types"; + +type ArchiveToCopyToContainer = { + filesToCopy?: FileToCopy[]; + directoriesToCopy?: DirectoryToCopy[]; + contentsToCopy?: ContentToCopy[]; +}; + +export async function createArchiveToCopyToContainer({ + filesToCopy = [], + directoriesToCopy = [], + contentsToCopy = [], +}: ArchiveToCopyToContainer): Promise { + const stagingDirectory = tmp.dirSync({ unsafeCleanup: true }); + + try { + for (const { source, target, mode } of filesToCopy) { + await copyFileToStagingDirectory(stagingDirectory.name, source, target, mode); + } + + for (const { source, target, mode } of directoriesToCopy) { + await copyDirectoryToStagingDirectory(stagingDirectory.name, source, target, mode); + } + + for (const { content, target, mode } of contentsToCopy) { + await copyContentToStagingDirectory(stagingDirectory.name, content, target, mode); + } + } catch (error) { + stagingDirectory.removeCallback(); + throw error; + } + + const archive = tar.pack(stagingDirectory.name, { dereference: true, umask: 0 }); + let cleanedUp = false; + const cleanup = () => { + if (cleanedUp) { + return; + } + + cleanedUp = true; + stagingDirectory.removeCallback(); + }; + + archive.once("end", cleanup); + archive.once("close", cleanup); + archive.once("error", cleanup); + + return archive; +} + +async function copyFileToStagingDirectory( + stagingDirectory: string, + source: string, + target: string, + mode: number | undefined +): Promise { + const targetPath = getArchiveTargetPath(stagingDirectory, target); + await fs.mkdir(path.dirname(targetPath), { recursive: true }); + await fs.cp(source, targetPath, { dereference: true }); + + if (mode !== undefined) { + await fs.chmod(targetPath, mode); + } +} + +async function copyDirectoryToStagingDirectory( + stagingDirectory: string, + source: string, + target: string, + mode: number | undefined +): Promise { + const targetPath = getArchiveTargetPath(stagingDirectory, target); + await fs.mkdir(targetPath, { recursive: true }); + + const entries = await fs.readdir(source); + await Promise.all( + entries.map((entry) => + fs.cp(path.resolve(source, entry), path.resolve(targetPath, entry), { recursive: true, dereference: true }) + ) + ); + + if (mode !== undefined) { + await setModeRecursively(targetPath, mode); + } +} + +async function copyContentToStagingDirectory( + stagingDirectory: string, + content: Content, + target: string, + mode: number | undefined +): Promise { + const targetPath = getArchiveTargetPath(stagingDirectory, target); + await fs.mkdir(path.dirname(targetPath), { recursive: true }); + await writeContentToFile(content, targetPath); + + if (mode !== undefined) { + await fs.chmod(targetPath, mode); + } +} + +async function writeContentToFile(content: Content, targetPath: string): Promise { + if (content instanceof Readable) { + await pipeline(content, createWriteStream(targetPath)); + } else { + await fs.writeFile(targetPath, content); + } +} + +async function setModeRecursively(targetPath: string, mode: number): Promise { + await fs.chmod(targetPath, mode); + + const entries = await fs.readdir(targetPath, { withFileTypes: true }); + await Promise.all( + entries.map(async (entry) => { + const entryPath = path.resolve(targetPath, entry.name); + + if (entry.isDirectory()) { + await setModeRecursively(entryPath, mode); + } else { + await fs.chmod(entryPath, mode); + } + }) + ); +} + +function getArchiveTargetPath(stagingDirectory: string, target: string): string { + const normalizedTarget = path.posix.resolve("/", target.replace(/\\/gu, "/")); + const relativeTarget = normalizedTarget.slice(1); + + if (relativeTarget.length === 0) { + return stagingDirectory; + } + + return path.resolve(stagingDirectory, ...relativeTarget.split("/")); +} diff --git a/packages/testcontainers/src/generic-container/generic-container.test.ts b/packages/testcontainers/src/generic-container/generic-container.test.ts index a5ebf3380..661b1e41f 100644 --- a/packages/testcontainers/src/generic-container/generic-container.test.ts +++ b/packages/testcontainers/src/generic-container/generic-container.test.ts @@ -1,6 +1,8 @@ -import archiver from "archiver"; +import { promises as fs } from "fs"; import getPort from "get-port"; import path from "path"; +import tar from "tar-fs"; +import tmp from "tmp"; import { RandomUuid } from "../common"; import { getContainerRuntimeClient } from "../container-runtime"; import { PullPolicy } from "../utils/pull-policy"; @@ -15,6 +17,41 @@ import { import { Wait } from "../wait-strategies/wait"; import { GenericContainer } from "./generic-container"; +async function createArchiveWithOwnership(target: string, uid: number, gid: number) { + const stagingDirectory = tmp.dirSync({ unsafeCleanup: true }); + + try { + const fileName = "archive.txt"; + await fs.writeFile(path.resolve(stagingDirectory.name, fileName), "hello world"); + const archiveEntryName = target.startsWith("/") ? target.slice(1) : target; + + const archive = tar.pack(stagingDirectory.name, { + entries: [fileName], + umask: 0, + map: (header) => ({ ...header, name: archiveEntryName, uid, gid }), + }); + + let cleanedUp = false; + const cleanup = () => { + if (cleanedUp) { + return; + } + + cleanedUp = true; + stagingDirectory.removeCallback(); + }; + + archive.once("end", cleanup); + archive.once("close", cleanup); + archive.once("error", cleanup); + + return archive; + } catch (error) { + stagingDirectory.removeCallback(); + throw error; + } +} + describe("GenericContainer", { timeout: 180_000 }, () => { const fixtures = path.resolve(__dirname, "..", "..", "fixtures", "docker"); @@ -521,9 +558,7 @@ describe("GenericContainer", { timeout: 180_000 }, () => { .withExposedPorts(8080) .start(); - const tar = archiver("tar"); - tar.append("hello world", { name: targetWithCopyOwnership.slice(1), uid, gid } as archiver.EntryData); - tar.finalize(); + const tar = await createArchiveWithOwnership(targetWithCopyOwnership, uid, gid); await container.copyArchiveToContainer(tar, "/", { copyUIDGID: true }); @@ -536,9 +571,7 @@ describe("GenericContainer", { timeout: 180_000 }, () => { const uid = 4242; const gid = 4343; const targetWithCopyOwnership = "/tmp/with-copy-archives-copyuidgid.txt"; - const tar = archiver("tar"); - tar.append("hello world", { name: targetWithCopyOwnership.slice(1), uid, gid } as archiver.EntryData); - tar.finalize(); + const tar = await createArchiveWithOwnership(targetWithCopyOwnership, uid, gid); await using containerWithCopyOwnership = await new GenericContainer("cristianrgreco/testcontainer:1.1.14") .withCopyArchivesToContainer([ diff --git a/packages/testcontainers/src/generic-container/generic-container.ts b/packages/testcontainers/src/generic-container/generic-container.ts index 937d29732..0cc5f291f 100644 --- a/packages/testcontainers/src/generic-container/generic-container.ts +++ b/packages/testcontainers/src/generic-container/generic-container.ts @@ -1,7 +1,5 @@ -import archiver from "archiver"; import AsyncLock from "async-lock"; import { Container, ContainerCreateOptions, HostConfig } from "dockerode"; -import { promises as fs } from "fs"; import { Readable } from "stream"; import { containerLog, hash, log, toNanos } from "../common"; import { ContainerRuntimeClient, getContainerRuntimeClient, ImageName } from "../container-runtime"; @@ -34,6 +32,7 @@ import { ImagePullPolicy, PullPolicy } from "../utils/pull-policy"; import { Wait } from "../wait-strategies/wait"; import { waitForContainer } from "../wait-strategies/wait-for-container"; import { WaitStrategy } from "../wait-strategies/wait-strategy"; +import { createArchiveToCopyToContainer } from "./create-archive-to-copy-to-container"; import { GenericContainerBuilder } from "./generic-container-builder"; import { inspectContainerUntilPortsExposed } from "./inspect-container-util-ports-exposed"; import { StartedGenericContainer } from "./started-generic-container"; @@ -182,8 +181,11 @@ export class GenericContainer implements TestContainer { } if (this.filesToCopy.length > 0 || this.directoriesToCopy.length > 0 || this.contentsToCopy.length > 0) { - const archive = await this.createArchiveToCopyToContainer(); - archive.finalize(); + const archive = await createArchiveToCopyToContainer({ + filesToCopy: this.filesToCopy, + directoriesToCopy: this.directoriesToCopy, + contentsToCopy: this.contentsToCopy, + }); await client.container.putArchive(container, archive, "/", this.copyToContainerOptions); } @@ -258,28 +260,6 @@ export class GenericContainer implements TestContainer { } } - private async createArchiveToCopyToContainer(): Promise { - const tar = archiver("tar"); - const filesToCopyWithStats = await Promise.all( - this.filesToCopy.map(async (fileToCopy) => ({ - ...fileToCopy, - stats: await fs.stat(fileToCopy.source), - })) - ); - - for (const { source, target, mode, stats } of filesToCopyWithStats) { - tar.file(source, { name: target, mode, stats }); - } - for (const { source, target, mode } of this.directoriesToCopy) { - tar.directory(source, target, { mode }); - } - for (const { content, target, mode } of this.contentsToCopy) { - tar.append(content, { name: target, mode }); - } - - return tar; - } - protected containerStarted?( container: StartedTestContainer, inspectResult: InspectResult, diff --git a/packages/testcontainers/src/generic-container/started-generic-container.ts b/packages/testcontainers/src/generic-container/started-generic-container.ts index 1b9c48421..81fa3c97b 100644 --- a/packages/testcontainers/src/generic-container/started-generic-container.ts +++ b/packages/testcontainers/src/generic-container/started-generic-container.ts @@ -1,7 +1,5 @@ -import archiver from "archiver"; import AsyncLock from "async-lock"; import Dockerode, { ContainerInspectInfo } from "dockerode"; -import { promises as fs } from "fs"; import { Readable } from "stream"; import { containerLog, log } from "../common"; import { ContainerRuntimeClient, getContainerRuntimeClient } from "../container-runtime"; @@ -23,6 +21,7 @@ import { mapInspectResult } from "../utils/map-inspect-result"; import { PortWithOptionalBinding } from "../utils/port"; import { waitForContainer } from "../wait-strategies/wait-for-container"; import { WaitStrategy } from "../wait-strategies/wait-strategy"; +import { createArchiveToCopyToContainer } from "./create-archive-to-copy-to-container"; import { inspectContainerUntilPortsExposed } from "./inspect-container-util-ports-exposed"; import { StoppedGenericContainer } from "./stopped-generic-container"; @@ -192,36 +191,24 @@ export class StartedGenericContainer implements StartedTestContainer { public async copyFilesToContainer(filesToCopy: FileToCopy[]): Promise { log.debug(`Copying files to container...`, { containerId: this.container.id }); const client = await getContainerRuntimeClient(); - const tar = archiver("tar"); - const filesToCopyWithStats = await Promise.all( - filesToCopy.map(async (fileToCopy) => ({ - ...fileToCopy, - stats: await fs.stat(fileToCopy.source), - })) - ); - filesToCopyWithStats.forEach(({ source, target, mode, stats }) => tar.file(source, { name: target, mode, stats })); - tar.finalize(); - await client.container.putArchive(this.container, tar, "/"); + const archive = await createArchiveToCopyToContainer({ filesToCopy }); + await client.container.putArchive(this.container, archive, "/"); log.debug(`Copied files to container`, { containerId: this.container.id }); } public async copyDirectoriesToContainer(directoriesToCopy: DirectoryToCopy[]): Promise { log.debug(`Copying directories to container...`, { containerId: this.container.id }); const client = await getContainerRuntimeClient(); - const tar = archiver("tar"); - directoriesToCopy.forEach(({ source, target }) => tar.directory(source, target)); - tar.finalize(); - await client.container.putArchive(this.container, tar, "/"); + const archive = await createArchiveToCopyToContainer({ directoriesToCopy }); + await client.container.putArchive(this.container, archive, "/"); log.debug(`Copied directories to container`, { containerId: this.container.id }); } public async copyContentToContainer(contentsToCopy: ContentToCopy[]): Promise { log.debug(`Copying content to container...`, { containerId: this.container.id }); const client = await getContainerRuntimeClient(); - const tar = archiver("tar"); - contentsToCopy.forEach(({ content, target, mode }) => tar.append(content, { name: target, mode: mode })); - tar.finalize(); - await client.container.putArchive(this.container, tar, "/"); + const archive = await createArchiveToCopyToContainer({ contentsToCopy }); + await client.container.putArchive(this.container, archive, "/"); log.debug(`Copied content to container`, { containerId: this.container.id }); } From 0d0f7f80a9b68aa8a0e9367acabb32ffccfbfd7a Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Sun, 22 Feb 2026 20:05:52 +0000 Subject: [PATCH 2/3] Fix tar-fs umask typing for build --- .../generic-container/create-archive-to-copy-to-container.ts | 2 +- .../src/generic-container/generic-container.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/testcontainers/src/generic-container/create-archive-to-copy-to-container.ts b/packages/testcontainers/src/generic-container/create-archive-to-copy-to-container.ts index e54692f3e..64e65af37 100644 --- a/packages/testcontainers/src/generic-container/create-archive-to-copy-to-container.ts +++ b/packages/testcontainers/src/generic-container/create-archive-to-copy-to-container.ts @@ -36,7 +36,7 @@ export async function createArchiveToCopyToContainer({ throw error; } - const archive = tar.pack(stagingDirectory.name, { dereference: true, umask: 0 }); + const archive = tar.pack(stagingDirectory.name, { dereference: true, umask: 0 } as tar.PackOptions); let cleanedUp = false; const cleanup = () => { if (cleanedUp) { diff --git a/packages/testcontainers/src/generic-container/generic-container.test.ts b/packages/testcontainers/src/generic-container/generic-container.test.ts index 661b1e41f..70ca878ce 100644 --- a/packages/testcontainers/src/generic-container/generic-container.test.ts +++ b/packages/testcontainers/src/generic-container/generic-container.test.ts @@ -29,7 +29,7 @@ async function createArchiveWithOwnership(target: string, uid: number, gid: numb entries: [fileName], umask: 0, map: (header) => ({ ...header, name: archiveEntryName, uid, gid }), - }); + } as tar.PackOptions); let cleanedUp = false; const cleanup = () => { From e3726bcc087646ef04a2a7e93342def8d1635267 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Sun, 22 Feb 2026 20:15:07 +0000 Subject: [PATCH 3/3] Normalize copy mode values for tar staging --- .../create-archive-to-copy-to-container.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/testcontainers/src/generic-container/create-archive-to-copy-to-container.ts b/packages/testcontainers/src/generic-container/create-archive-to-copy-to-container.ts index 64e65af37..cad2f3633 100644 --- a/packages/testcontainers/src/generic-container/create-archive-to-copy-to-container.ts +++ b/packages/testcontainers/src/generic-container/create-archive-to-copy-to-container.ts @@ -65,7 +65,7 @@ async function copyFileToStagingDirectory( await fs.cp(source, targetPath, { dereference: true }); if (mode !== undefined) { - await fs.chmod(targetPath, mode); + await fs.chmod(targetPath, normalizeArchiveMode(mode)); } } @@ -86,7 +86,7 @@ async function copyDirectoryToStagingDirectory( ); if (mode !== undefined) { - await setModeRecursively(targetPath, mode); + await setModeRecursively(targetPath, normalizeArchiveMode(mode)); } } @@ -101,7 +101,7 @@ async function copyContentToStagingDirectory( await writeContentToFile(content, targetPath); if (mode !== undefined) { - await fs.chmod(targetPath, mode); + await fs.chmod(targetPath, normalizeArchiveMode(mode)); } } @@ -140,3 +140,15 @@ function getArchiveTargetPath(stagingDirectory: string, target: string): string return path.resolve(stagingDirectory, ...relativeTarget.split("/")); } + +function normalizeArchiveMode(mode: number): number { + if (mode <= 0o777) { + return mode; + } + + if (mode <= 0o7777 && /^[0-7]+$/u.test(String(mode))) { + return parseInt(String(mode), 8); + } + + return mode; +}