Skip to content

fix(sdk): zip64/APPNOTE conformance in the TDF3 zip reader and writer (DSPX-4591) #1017

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dmihalcik-virtru wants to merge 1 commit into
base: main
Choose a base branch
from
+793 −36
Open

fix(sdk): zip64/APPNOTE conformance in the TDF3 zip reader and writer (DSPX-4591) #1017

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
259 changes: 228 additions & 31 deletions lib/tdf3/src/utils/zip-reader.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,51 @@ import { type Chunker } from '../../../src/seekable.js';
import { Manifest } from '../models/index.js';
import { readUInt32LE, readUInt16LE, copyUint8Arr, buffToString } from './index.js';

// TODO: Better document what these constants are
// TODO: Document each function please
// Signatures and fixed record sizes from PKWARE APPNOTE.TXT sections 4.3.12-4.3.16.
/** Central file header signature (APPNOTE 4.3.12). */
const CD_SIGNATURE = 0x02014b50;
/** End of central directory record signature (APPNOTE 4.3.16). */
const EOCDR_SIGNATURE = 0x06054b50;
/** ZIP64 end of central directory record signature (APPNOTE 4.3.14). */
const ZIP64_EOCDR_SIGNATURE = 0x06064b50;
/** ZIP64 end of central directory locator signature (APPNOTE 4.3.15). */
const ZIP64_EOCDL_SIGNATURE = 0x07064b50;

/** Size of a central file header, excluding name, extra field and comment. */
const CENTRAL_DIRECTORY_RECORD_FIXED_SIZE = 46;
/** Size of a local file header, excluding name and extra field. */
const LOCAL_FILE_HEADER_FIXED_SIZE = 30;
const VERSION_NEEDED_TO_EXTRACT_ZIP64 = 45;
/** Size of an end of central directory record, excluding the archive comment. */
const END_OF_CENTRAL_DIRECTORY_RECORD_SIZE = 22;
/** Size of a ZIP64 end of central directory locator. Fixed length. */
const ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR_SIZE = 20;
/** Bytes of the ZIP64 end of central directory record that we read and rely on. */
const ZIP64_END_OF_CENTRAL_DIRECTORY_RECORD_SIZE = 56;

/** The archive comment length field is 2 bytes, so a comment is at most 64 KiB. */
const MAX_ARCHIVE_COMMENT_SIZE = 0xffff;
/**
* Enough bytes to always contain the EOCD record, its (optional) ZIP64 locator,
* and a maximum length archive comment.
*/
const MAX_EOCDR_SEARCH_SIZE =
END_OF_CENTRAL_DIRECTORY_RECORD_SIZE +
MAX_ARCHIVE_COMMENT_SIZE +
ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR_SIZE;
/**
* First guess at how much of the tail to read. TDF containers are written without
* an archive comment, so the EOCD is the last 22 bytes and one small read suffices.
* If the EOCD is not in here we fall back to {@link MAX_EOCDR_SEARCH_SIZE}.
*/
const INITIAL_EOCDR_SEARCH_SIZE = 1024;

/**
* Sanity bound on the central directory we are willing to buffer. At 46 bytes per
* record this still allows for hundreds of thousands of entries, while keeping a
* hostile or corrupt EOCD from asking us to allocate the declared 4 GiB.
*/
const MAX_CENTRAL_DIRECTORY_SIZE = 16 * 1024 * 1024;

const manifestMaxSize = 1024 * 1024 * 10; // 10 MB

const cp437 =
Expand Down Expand Up @@ -53,6 +92,22 @@ export type CentralDirectoryVariableLengthItems = {
headerLength: number;
};

/**
* The parts of the end of central directory record (APPNOTE 4.3.16), with the
* ZIP64 end of central directory record (APPNOTE 4.3.14) already applied where
* the 32 bit record carried a sentinel value.
*/
export type EndOfCentralDirectory = {
/** Total number of central directory records. */
entryCount: number;
/** Size in bytes of the central directory. */
centralDirectorySize: number;
/** Offset of the first central directory record from the start of the archive. */
centralDirectoryOffset: number;
/** True if the values above were read from a ZIP64 end of central directory record. */
zip64: boolean;
};

/**
*
* ZipReader -
Expand All @@ -69,22 +124,118 @@ export class ZipReader {

/**
* Utility function to get the centralDirectory for the zip file.
* It reads the end of the file to find it.
*
* Reads the end of central directory record (following the ZIP64 locator when
* the EOCD carries a sentinel), then walks exactly the declared number of
* records starting at the declared central directory offset.
*
* @return The central directory represented as an object
*/
async getCentralDirectory(): Promise<CentralDirectory[]> {
const chunk = await this.getChunk(-1000);
// TODO: Does this need to be tuned??!?
// Slice off the EOCDR (End of Central Directory Record) part of the buffer so we can figure out the CD size
const cdBuffers = this.getCDBuffers(chunk);
const eocd = await this.getEndOfCentralDirectory();
const cdChunk = await this.getChunk(
eocd.centralDirectoryOffset,
eocd.centralDirectoryOffset + eocd.centralDirectorySize
);
if (cdChunk.length !== eocd.centralDirectorySize) {
throw new InvalidFileError(
`central directory truncated: expected [${eocd.centralDirectorySize}] bytes at [${eocd.centralDirectoryOffset}], read [${cdChunk.length}]`
);
}

const cdParsedBuffers = cdBuffers.map(parseCDBuffer);
const cdParsedBuffers = this.getCDBuffers(cdChunk, eocd.entryCount).map(parseCDBuffer);
for (const buffer of cdParsedBuffers) {
await this.adjustHeaders(buffer);
}
return cdParsedBuffers;
}

/**
* Locates and parses the end of central directory record, resolving it against
* the ZIP64 end of central directory record when any of the three EOCD fields
* carries its sentinel value (APPNOTE 4.3.14 - 4.3.16).
*/
async getEndOfCentralDirectory(): Promise<EndOfCentralDirectory> {
let tail = await this.getChunk(-INITIAL_EOCDR_SEARCH_SIZE);
let eocdrOffset = findEndOfCentralDirectoryRecord(tail);
if (eocdrOffset < 0 && tail.length >= INITIAL_EOCDR_SEARCH_SIZE) {
// The archive may carry a comment of up to 64 KiB; widen the search once.
tail = await this.getChunk(-MAX_EOCDR_SEARCH_SIZE);
eocdrOffset = findEndOfCentralDirectoryRecord(tail);
}
if (eocdrOffset < 0) {
throw new InvalidFileError('unable to find end of central directory record');
}

// 8 - total number of entries in the central directory on this disk (2 bytes)
// 10 - total number of entries in the central directory (2 bytes)
let entryCount = readUInt16LE(tail, eocdrOffset + 10);
// 12 - size of the central directory (4 bytes)
let centralDirectorySize = readUInt32LE(tail, eocdrOffset + 12);
// 16 - offset of start of central directory (4 bytes)
let centralDirectoryOffset = readUInt32LE(tail, eocdrOffset + 16);

const needsZip64 =
entryCount === 0xffff ||
centralDirectorySize === 0xffffffff ||
centralDirectoryOffset === 0xffffffff;
if (needsZip64) {
const zip64 = await this.getZip64EndOfCentralDirectory(tail, eocdrOffset);
({ entryCount, centralDirectorySize, centralDirectoryOffset } = zip64);
}

if (centralDirectorySize > MAX_CENTRAL_DIRECTORY_SIZE) {
throw new InvalidFileError(
`central directory too large: [${centralDirectorySize}] bytes exceeds [${MAX_CENTRAL_DIRECTORY_SIZE}]`
);
}
if (centralDirectorySize < entryCount * CENTRAL_DIRECTORY_RECORD_FIXED_SIZE) {
throw new InvalidFileError(
`central directory size [${centralDirectorySize}] too small for [${entryCount}] entries`
);
}

return { entryCount, centralDirectorySize, centralDirectoryOffset, zip64: needsZip64 };
}

/**
* Follows the ZIP64 end of central directory locator, which sits immediately
* before the EOCD record, and reads the ZIP64 EOCD record it points at.
*/
private async getZip64EndOfCentralDirectory(
tail: Uint8Array,
eocdrOffset: number
): Promise<Omit<EndOfCentralDirectory, 'zip64'>> {
const locatorOffset = eocdrOffset - ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR_SIZE;
if (locatorOffset < 0 || readUInt32LE(tail, locatorOffset) !== ZIP64_EOCDL_SIGNATURE) {
throw new InvalidFileError(
'end of central directory record requires zip64, but no zip64 locator was found'
);
}
// 8 - relative offset of the zip64 end of central directory record (8 bytes)
const zip64EocdrOffset = readUInt64LE(tail, locatorOffset + 8);
const record = await this.getChunk(
zip64EocdrOffset,
zip64EocdrOffset + ZIP64_END_OF_CENTRAL_DIRECTORY_RECORD_SIZE
);
if (
record.length < ZIP64_END_OF_CENTRAL_DIRECTORY_RECORD_SIZE ||
readUInt32LE(record, 0) !== ZIP64_EOCDR_SIGNATURE
) {
throw new InvalidFileError(
`invalid zip64 end of central directory record at [${zip64EocdrOffset}]`
);
}
return {
// 32 - total number of entries in the central directory (8 bytes)
entryCount: readUInt64LE(record, 32),
// 40 - size of the central directory (8 bytes)
centralDirectorySize: readUInt64LE(record, 40),
// 48 - offset of start of central directory (8 bytes)
centralDirectoryOffset: readUInt64LE(record, 48),
};
}

/**
* Gets the manifest
* @returns The manifest as a buffer represented as JSON
Expand All @@ -97,7 +248,7 @@ export class ZipReader {
const byteStart = cdObj.relativeOffsetOfLocalHeader + cdObj.headerLength;
if (cdObj.uncompressedSize > manifestMaxSize) {
throw new InvalidFileError(
`manifest file too large: ${(cdObj.uncompressedSize >> 10).toLocaleString()} KiB`
`manifest file too large: ${Math.floor(cdObj.uncompressedSize / 1024).toLocaleString()} KiB`
);
}
const byteEnd = byteStart + cdObj.uncompressedSize;
Expand Down Expand Up @@ -137,31 +288,70 @@ export class ZipReader {
}

/**
* extracts the CD buffer entries from the end of a zip file.
* @param chunkBuffer The last portion of a zip file
* Splits the central directory into its individual records.
*
* Boundaries come from each record's own declared name, extra field and comment
* lengths - not from scanning for the next signature - and exactly `entryCount`
* records are read, as declared by the end of central directory record.
*
* @param cdChunk the central directory, starting at its first record
* @param entryCount the number of records the EOCD says are present
* @returns an array of typed arrays, each element corresponding to a central directory record
*/
getCDBuffers(chunkBuffer: Uint8Array): Uint8Array[] {
const cdBuffers = [];
let lastBufferOffset = chunkBuffer.length;
for (let i = chunkBuffer.length - 22; i >= 0; i -= 1) {
// If what we're locking at isn't the start of a central directory, skip it..
if (readUInt32LE(chunkBuffer, i) !== CD_SIGNATURE) {
// eslint-disable-next-line no-continue
continue;
getCDBuffers(cdChunk: Uint8Array, entryCount: number): Uint8Array[] {
const cdBuffers: Uint8Array[] = [];
let offset = 0;
for (let i = 0; i < entryCount; i++) {
if (offset + CENTRAL_DIRECTORY_RECORD_FIXED_SIZE > cdChunk.length) {
throw new InvalidFileError(
`central directory ended early: found [${i}] of [${entryCount}] declared entries`
);
}
// Slice off that CD from it's start until the end of either the buffer, or whatever the start of the previously
// found CD was
cdBuffers.push(chunkBuffer.slice(i, lastBufferOffset));
// Store the last offset location so we know how to slice off hte next CD.
lastBufferOffset = i;
// We can skip over 22 iterations since we know the minimum size of a CD is 22.
i -= 22;
if (readUInt32LE(cdChunk, offset) !== CD_SIGNATURE) {
throw new InvalidFileError(
`invalid central directory file header signature for entry [${i}]`
);
}
// 28 - file name length (n), 30 - extra field length (m), 32 - file comment length (k)
const fileNameLength = readUInt16LE(cdChunk, offset + 28);
const extraFieldLength = readUInt16LE(cdChunk, offset + 30);
const fileCommentLength = readUInt16LE(cdChunk, offset + 32);
const recordLength =
CENTRAL_DIRECTORY_RECORD_FIXED_SIZE + fileNameLength + extraFieldLength + fileCommentLength;
if (offset + recordLength > cdChunk.length) {
throw new InvalidFileError(
`central directory record [${i}] of length [${recordLength}] overruns the central directory`
);
}
cdBuffers.push(cdChunk.slice(offset, offset + recordLength));
offset += recordLength;
}
return cdBuffers;
}
}

// They should be in the correct order. Since we iterate backwards, it's built backwards.
return cdBuffers.reverse();
/**
* Scans a buffer whose final byte is the final byte of the archive for the end of
* central directory record.
*
* A candidate is only accepted when its declared comment length places the end of
* the comment exactly at the end of the archive, which rules out payload bytes that
* happen to spell the signature.
*
* @returns the index of the EOCD signature within `tail`, or -1 if not found
*/
function findEndOfCentralDirectoryRecord(tail: Uint8Array): number {
for (let i = tail.length - END_OF_CENTRAL_DIRECTORY_RECORD_SIZE; i >= 0; i--) {
if (readUInt32LE(tail, i) !== EOCDR_SIGNATURE) {
continue;
}
// 20 - .ZIP file comment length (2 bytes)
const commentLength = readUInt16LE(tail, i + 20);
if (i + END_OF_CENTRAL_DIRECTORY_RECORD_SIZE + commentLength === tail.length) {
return i;
}
}
return -1;
}

function parseCentralDirectoryWithNoExtras(cdBuffer: Uint8Array): CentralDirectory {
Expand Down Expand Up @@ -215,14 +405,21 @@ function parseCentralDirectoryWithNoExtras(cdBuffer: Uint8Array): CentralDirecto
* @return The CD object
*/
export function parseCDBuffer(cdBuffer: Uint8Array): CentralDirectory {
if (cdBuffer.length < CENTRAL_DIRECTORY_RECORD_FIXED_SIZE) {
throw new InvalidFileError('Truncated central directory file header');
}
if (readUInt32LE(cdBuffer, 0) !== CD_SIGNATURE) {
throw new InvalidFileError('Invalid central directory file header signature');
}

const cd = parseCentralDirectoryWithNoExtras(cdBuffer);

if (cd.versionNeededToExtract < VERSION_NEEDED_TO_EXTRACT_ZIP64 || !cd.extraFieldLength) {
// NOTE(PLAT-1134) Zip64 was added in pkzip 4.5
// NOTE(DSPX-4591): APPNOTE 4.5.3 does not condition the validity of a zip64
// extended information extra field on `version needed to extract`; the sentinel
// values in the fixed-size fields are what select it. Gating on the version byte
// silently dropped the extra field, leaving 0xffffffff to flow into offset and
// size arithmetic as if it were a real number.
if (!cd.extraFieldLength) {
return cd;
}

Expand Down
22 changes: 17 additions & 5 deletions lib/tdf3/src/utils/zip-writer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,20 @@ export class ZipWriter {
]);
}

writeDataDescriptor(crc32: number, uncompressedSize: number): Uint8Array {
/**
* Builds a data descriptor record (APPNOTE 4.3.9):
* `signature, crc-32, compressed size, uncompressed size`.
*
* @param crc32 CRC-32 of the uncompressed data
* @param uncompressedSize size of the entry before compression
* @param compressedSize size of the entry as stored; defaults to `uncompressedSize`
* because we only ever use the STORE method
*/
writeDataDescriptor(
crc32: number,
uncompressedSize: number,
compressedSize: number = uncompressedSize
): Uint8Array {
// NOTE(PLAT-1134): optional signature (required according to Archive Utility)
// 4.3.9.3 Although not originally assigned a signature, the value
// 0x08074b50 has commonly been adopted as a signature value
Expand All @@ -188,16 +201,15 @@ export class ZipWriter {
// the file the compressed and uncompressed sizes will be 8
// byte values.
buffer = new Uint8Array(ZIP64_DATA_DESCRIPTOR_SIZE);
writeUInt32LE(buffer, crc32, 4);
writeUInt32LE(buffer, ddSig, 0);
// We just use STORE, so compressed and uncompressed are the same.
writeUInt64LE(buffer, uncompressedSize, 8);
writeUInt32LE(buffer, crc32, 4);
writeUInt64LE(buffer, compressedSize, 8);
writeUInt64LE(buffer, uncompressedSize, 16);
} else {
buffer = new Uint8Array(DATA_DESCRIPTOR_SIZE);
writeUInt32LE(buffer, ddSig, 0);
writeUInt32LE(buffer, crc32, 4);
writeUInt32LE(buffer, uncompressedSize, 8);
writeUInt32LE(buffer, compressedSize, 8);
writeUInt32LE(buffer, uncompressedSize, 12);
}
return buffer;
Expand Down
Loading
Loading