base-data-manager/data-export/zipFs.ts

364 lines
11 KiB
TypeScript

import { strict as assert } from "node:assert";
import { Readable } from "node:stream";
import yauzl from "yauzl";
import { globSync } from "glob";
function removeDummyCwd(path: string) {
if (path.startsWith("/DUMMYCWD/")) {
// This is so we can properly call globSync with _some_ cwd
// and then strip it later, as a cwd of "" will use the current
// working directory of the process (which will matching nothing)
return path.slice("/DUMMYCWD/".length);
}
return path;
}
// Dirent-like class for directory entries
class ZipDirent {
name: string;
private isDir: boolean;
constructor(name: string, isDirectory: boolean) {
this.name = name;
this.isDir = isDirectory;
}
isFile(): boolean {
return !this.isDir;
}
isDirectory(): boolean {
return this.isDir;
}
isBlockDevice(): boolean {
return false;
}
isCharacterDevice(): boolean {
return false;
}
isSymbolicLink(): boolean {
return false;
}
isFIFO(): boolean {
return false;
}
isSocket(): boolean {
return false;
}
}
function _entryToStats(entry: yauzl.Entry) {
const isDir = entry.fileName.endsWith("/");
const modDate = entry.getLastModDate();
return {
isFile: () => !isDir,
isDirectory: () => isDir,
isBlockDevice: () => false,
isCharacterDevice: () => false,
isSymbolicLink: () => false,
isFIFO: () => false,
isSocket: () => false,
size: entry.uncompressedSize,
compressedSize: entry.compressedSize,
mtime: modDate,
mode: isDir ? 0o040755 : 0o100644,
uid: 0,
gid: 0,
dev: 0,
ino: 0,
nlink: 1,
rdev: 0,
blksize: 4096,
blocks: Math.ceil(entry.uncompressedSize / 512),
atime: modDate,
ctime: modDate,
birthtime: modDate,
atimeMs: modDate.getTime(),
mtimeMs: modDate.getTime(),
ctimeMs: modDate.getTime(),
birthtimeMs: modDate.getTime(),
};
}
export class ZipFS {
isZip = true;
zipPath: string;
entries: Map<string, yauzl.Entry>;
zipFile: yauzl.ZipFile | null;
constructor(path: string) {
this.zipPath = path;
this.entries = new Map();
this.zipFile = null;
}
async init() {
this.zipFile = await new Promise<yauzl.ZipFile>((resolve, reject) => {
yauzl.open(this.zipPath, {
lazyEntries: true,
autoClose: false
}, (err, zipfile) => {
if (err || !zipfile) return reject(err);
resolve(zipfile);
});
});
await new Promise<void>((resolve, reject) => {
this.zipFile!.readEntry();
this.zipFile!.on("entry", (entry) => {
const name = entry.fileName;
this.entries.set(name, entry);
this.zipFile!.readEntry();
});
this.zipFile!.on("end", resolve);
this.zipFile!.on("error", reject);
});
}
get ready() {
return !!this.zipFile;
}
existsSync(path: string): boolean {
assert(this.zipFile, 'Must be inited');
path = removeDummyCwd(path);
return this.entries.has(path);
}
stat(path: string) {
assert(this.zipFile, 'Must be inited');
path = removeDummyCwd(path);
const entry = this.entries.get(path);
if (!entry) throw new Error(`ENOENT: no such file or directory, stat '${path}'`);
return _entryToStats(entry);
}
statSync(path: string) {
assert(this.zipFile, 'Must be inited');
path = removeDummyCwd(path);
const entry = this.entries.get(path);
if (!entry) throw new Error(`ENOENT: no such file or directory, stat '${path}'`);
return _entryToStats(entry);
}
lstatSync(path: string) {
// ZIP files don't have symlinks, so lstat is the same as stat
path = removeDummyCwd(path);
return this.statSync(path);
}
createWriteStream(path: string): never {
throw new Error("ZIP filesystem is read-only");
}
createReadStream(path: string): Readable {
assert(this.zipFile, 'Must be inited');
path = removeDummyCwd(path);
const entry = this.entries.get(path);
if (!entry) throw new Error(`ENOENT: no such file or directory, open '${path}'`);
const out = new Readable({ read() {} });
this.zipFile.openReadStream(entry, (err, stream) => {
if (err || !stream) {
out.destroy(err ?? new Error("Failed to open stream"));
return;
}
stream.on("data", (chunk) => out.push(chunk));
stream.on("end", () => out.push(null));
stream.on("error", (e) => out.destroy(e));
});
return out;
}
private _listDirectory(dirPath: string): string[] {
// Normalize the directory path
let normalizedDir = dirPath.replace(/\\/g, "/");
if (normalizedDir && !normalizedDir.endsWith("/")) {
normalizedDir += "/";
}
const results = new Set<string>();
for (const entryPath of this.entries.keys()) {
// Check if this entry is directly under the directory
if (entryPath === normalizedDir) continue; // Skip the directory itself
if (normalizedDir === "" || normalizedDir === "/") {
// Root directory - get top-level entries
const parts = entryPath.split("/").filter(p => p);
if (parts.length > 0) {
results.add(parts[0]);
}
} else if (entryPath.startsWith(normalizedDir)) {
// Get the relative path from the directory
const relativePath = entryPath.substring(normalizedDir.length);
const parts = relativePath.split("/").filter(p => p);
if (parts.length > 0) {
results.add(parts[0]);
}
}
}
return Array.from(results).sort();
}
readdirSync(dirPath: string, options?: { withFileTypes?: false }): string[];
readdirSync(dirPath: string, options: { withFileTypes: true }): ZipDirent[];
readdirSync(dirPath: string, options?: { withFileTypes?: boolean }): string[] | ZipDirent[] {
assert(this.zipFile, 'Must be inited');
dirPath = removeDummyCwd(dirPath);
const entries = this._listDirectory(dirPath);
if (options?.withFileTypes) {
return entries.map(name => {
let fullPath = dirPath.replace(/\\/g, "/");
if (fullPath && !fullPath.endsWith("/")) {
fullPath += "/";
}
const entryPath = fullPath + name;
// Check if it's a directory by looking for entries with this prefix
const isDirectory = this.entries.has(entryPath + "/") ||
Array.from(this.entries.keys()).some(p => p.startsWith(entryPath + "/"));
return new ZipDirent(name, isDirectory);
});
}
return entries;
}
async readdir(dirPath: string, options?: { withFileTypes?: false }): Promise<string[]>;
async readdir(dirPath: string, options: { withFileTypes: true }): Promise<ZipDirent[]>;
async readdir(dirPath: string, options?: { withFileTypes?: boolean }): Promise<string[] | ZipDirent[]> {
assert(this.zipFile, 'Must be inited');
dirPath = removeDummyCwd(dirPath);
return this.readdirSync(dirPath, options as any);
}
readlinkSync(path: string): string {
// ZIP files don't support symlinks
throw new Error(`EINVAL: invalid argument, readlink '${path}'`);
}
realpathSync(path: string): string {
assert(this.zipFile, 'Must be inited');
path = removeDummyCwd(path);
// Normalize the path and check if it exists
const normalized = path.replace(/\\/g, "/");
if (this.entries.has(normalized) || this.entries.has(normalized + "/")) {
return normalized;
}
// Check if it's a valid directory path
const withSlash = normalized.endsWith("/") ? normalized : normalized + "/";
const hasChildren = Array.from(this.entries.keys()).some(p => p.startsWith(withSlash));
if (hasChildren) {
return normalized;
}
throw new Error(`ENOENT: no such file or directory, realpath '${path}'`);
}
promises = {
lstat: async (path: string) => {
return this.lstatSync(path);
},
readdir: async (dirPath: string, options?: { withFileTypes?: false }): Promise<string[]> => {
return this.readdirSync(dirPath, options);
},
readlink: async (path: string): Promise<string> => {
return this.readlinkSync(path);
},
realpath: async (path: string): Promise<string> => {
return this.realpathSync(path);
},
};
lstat(path: string, callback: (err: Error | null, stats?: any) => void) {
assert(this.zipFile, 'Must be inited');
path = removeDummyCwd(path);
try {
const stats = this.lstatSync(path);
callback(null, stats);
} catch (err) {
callback(err as Error);
}
}
realpath(path: string, callback: (err: Error | null, resolvedPath?: string) => void) {
assert(this.zipFile, 'Must be inited');
path = removeDummyCwd(path);
try {
const resolved = this.realpathSync(path);
callback(null, resolved);
} catch (err) {
callback(err as Error);
}
}
globSync(globPath: string) {
const selfImpl = this.getImpl();
return globSync(globPath, {
fs: selfImpl as any,
// We strip this later, this is so glob() doesn't use the cwd of the current
// process, which matches no files inside the .zip file
cwd: `/DUMMYCWD`
});
}
getImpl() {
// Because glob uses ...xxx notation to unpack ourselves into a _new_ object
// we need to make sure that we DONT use a class, otherwise the properties
// will be non-enumerable and not show up in the output object
return {
isZip: this.isZip,
zipPath: this.zipPath,
init: this.init.bind(this),
ready: this.ready,
globSync: this.globSync.bind(this),
statSync: this.statSync.bind(this),
createReadStream: this.createReadStream.bind(this),
createWriteStream: this.createWriteStream.bind(this),
existsSync: this.existsSync.bind(this),
lstatSync: this.lstatSync.bind(this),
readdir: this.readdir.bind(this),
readdirSync: this.readdirSync.bind(this),
readlinkSync: this.readlinkSync.bind(this),
realpathSync: this.realpathSync.bind(this),
};
}
access() { throw new Error("Not implemented"); }
appendFile() { throw new Error("Not implemented"); }
chmod() { throw new Error("Not implemented"); }
chown() { throw new Error("Not implemented"); }
copyFile() { throw new Error("Not implemented"); }
mkdir() { throw new Error("Not implemented"); }
mkdtemp() { throw new Error("Not implemented"); }
open() { throw new Error("Not implemented"); }
readFile() { throw new Error("Not implemented"); }
rename() { throw new Error("Not implemented"); }
rm() { throw new Error("Not implemented"); }
rmdir() { throw new Error("Not implemented"); }
statfs() { throw new Error("Not implemented"); }
symlink() { throw new Error("Not implemented"); }
truncate() { throw new Error("Not implemented"); }
unlink() { throw new Error("Not implemented"); }
utimes() { throw new Error("Not implemented"); }
watch() { throw new Error("Not implemented"); }
writeFile() { throw new Error("Not implemented"); }
}