1export interface FieldSchema {2 readonly id: string;3 readonly name: string;4 readonly kind: number;5 readonly optional?: boolean;6}78export interface TypeSchema {9 readonly id: string;10 readonly name: string;11 readonly fields: readonly FieldSchema[];12}1314export interface Schema {15 readonly types: readonly TypeSchema[];16}1718export interface FieldProjection {19 readonly known: readonly string[];20 readonly foreign: readonly string[];21}2223const HEADER_SIZE = 80;24const DIRECTORY_SIZE = 64;25const MAGIC = "ATLASPK\0";2627interface ColumnDirectory {28 readonly id: string;29 readonly kind: number;30 readonly optional: boolean;31 readonly present: Uint8Array;32 readonly payload: Uint8Array;33}3435export class PackedColumn {36 readonly id: string;37 readonly kind: number;38 readonly optional: boolean;39 readonly #rows: number;40 readonly #present: Uint8Array;41 readonly #payload: Uint8Array;42 readonly #view: DataView;43 readonly #decoder = new TextDecoder();4445 constructor(rows: number, directory: ColumnDirectory) {46 this.id = directory.id;47 this.kind = directory.kind;48 this.optional = directory.optional;49 this.#rows = rows;50 this.#present = directory.present;51 this.#payload = directory.payload;52 this.#view = new DataView(directory.payload.buffer, directory.payload.byteOffset, directory.payload.byteLength);53 this.#validateShape();54 }5556 isNull(row: number): boolean {57 this.#checkRow(row);58 const byte = this.#present[Math.floor(row / 8)];59 return byte === undefined || (byte & (1 << (row % 8))) === 0;60 }6162 bool(row: number): boolean | null {63 this.#expect(1);64 return this.isNull(row) ? null : this.#payload[row] !== 0;65 }6667 int64(row: number): bigint | null {68 this.#expect(2);69 return this.isNull(row) ? null : this.#view.getBigInt64(row * 8, true);70 }7172 float64(row: number): number | null {73 this.#expect(3);74 return this.isNull(row) ? null : this.#view.getFloat64(row * 8, true);75 }7677 string(row: number): string | null {78 this.#expect(4);79 const value = this.#variable(row);80 return value === null ? null : this.#decoder.decode(value);81 }8283 bytes(row: number): Uint8Array | null {84 this.#expect(5);85 return this.#variable(row);86 }8788 identifier(row: number): string | null {89 this.#expect(6);90 if (this.isNull(row)) return null;91 return formatID(this.#payload.subarray(row * 16, (row + 1) * 16));92 }9394 #variable(row: number): Uint8Array | null {95 if (this.isNull(row)) return null;96 const offsetsLength = (this.#rows + 1) * 4;97 const start = this.#view.getUint32(row * 4, true);98 const end = this.#view.getUint32((row + 1) * 4, true);99 return this.#payload.subarray(offsetsLength + start, offsetsLength + end);100 }101102 #validateShape(): void {103 const expected = this.kind === 1 ? this.#rows104 : this.kind === 2 || this.kind === 3 ? this.#rows * 8105 : this.kind === 6 ? this.#rows * 16106 : undefined;107 if (expected !== undefined && this.#payload.byteLength !== expected) throw new Error(`column ${this.id} has an invalid payload length`);108 if (this.kind === 4 || this.kind === 5) {109 const offsetsLength = (this.#rows + 1) * 4;110 if (this.#payload.byteLength < offsetsLength) throw new Error(`column ${this.id} has truncated offsets`);111 let previous = 0;112 const contentLength = this.#payload.byteLength - offsetsLength;113 for (let row = 0; row <= this.#rows; row++) {114 const offset = this.#view.getUint32(row * 4, true);115 if (offset < previous || offset > contentLength) throw new Error(`column ${this.id} has invalid offsets`);116 previous = offset;117 }118 }119 }120121 #expect(kind: number): void {122 if (this.kind !== kind) throw new Error(`column ${this.id} has wire kind ${this.kind}, not ${kind}`);123 }124125 #checkRow(row: number): void {126 if (!Number.isInteger(row) || row < 0 || row >= this.#rows) throw new RangeError(`row ${row} is outside the table`);127 }128}129130export class PackedTable {131 readonly rows: number;132 readonly rootType: string;133 readonly schemaHash: string;134 readonly columns: readonly PackedColumn[];135 readonly #byID: ReadonlyMap<string, PackedColumn>;136137 private constructor(rows: number, rootType: string, schemaHash: string, columns: readonly PackedColumn[]) {138 this.rows = rows;139 this.rootType = rootType;140 this.schemaHash = schemaHash;141 this.columns = columns;142 this.#byID = new Map(columns.map((column) => [column.id, column]));143 }144145 static over(source: ArrayBuffer | Uint8Array): PackedTable {146 const bytes = source instanceof Uint8Array ? source : new Uint8Array(source);147 const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);148 if (bytes.byteLength < HEADER_SIZE || new TextDecoder().decode(bytes.subarray(0, 8)) !== MAGIC) throw new Error("typed block has no Atlas magic");149 if (view.getUint16(8, true) !== 1) throw new Error("typed block framing is unsupported");150 const rows = view.getUint32(12, true);151 const count = view.getUint32(16, true);152 const directoryEnd = HEADER_SIZE + count * DIRECTORY_SIZE;153 if (view.getUint32(68, true) !== DIRECTORY_SIZE || view.getBigUint64(72, true) !== BigInt(directoryEnd) || directoryEnd > bytes.byteLength) {154 throw new Error("typed block directory framing is invalid");155 }156 const columns: PackedColumn[] = [];157 for (let index = 0; index < count; index++) {158 const at = HEADER_SIZE + index * DIRECTORY_SIZE;159 const kind = bytes[at + 16];160 if (kind === undefined || kind < 1 || kind > 6 || bytes[at + 17] !== kind) throw new Error("column has unsupported encoding");161 const present = boundedSlice(bytes, view.getBigUint64(at + 20, true), view.getBigUint64(at + 28, true));162 const payload = boundedSlice(bytes, view.getBigUint64(at + 36, true), view.getBigUint64(at + 44, true));163 if (present.byteLength !== Math.ceil(rows / 8)) throw new Error("column presence bitmap is invalid");164 if (crc32(present, payload) !== view.getUint32(at + 52, true)) throw new Error("column checksum does not match");165 columns.push(new PackedColumn(rows, {166 id: formatID(bytes.subarray(at, at + 16)), kind,167 optional: (view.getUint16(at + 18, true) & 1) !== 0, present, payload,168 }));169 }170 return new PackedTable(rows, formatID(bytes.subarray(52, 68)), hex(bytes.subarray(20, 52)), columns);171 }172173 column(id: string): PackedColumn | undefined { return this.#byID.get(normalizeID(id)); }174}175176export function mergeSchemas(...schemas: readonly Schema[]): Schema {177 const types = new Map<string, { id: string; name: string; fields: Map<string, FieldSchema> }>();178 for (const schema of schemas) {179 for (const incoming of schema.types) {180 const typeID = normalizeID(incoming.id);181 const current = types.get(typeID) ?? { id: typeID, name: incoming.name, fields: new Map<string, FieldSchema>() };182 if (incoming.name < current.name) current.name = incoming.name;183 for (const field of incoming.fields) {184 const fieldID = normalizeID(field.id);185 const held = current.fields.get(fieldID);186 if (held && held.kind !== field.kind) throw new Error(`field ${fieldID} changes wire kind from ${held.kind} to ${field.kind}`);187 const name = held && held.name < field.name ? held.name : field.name;188 const optional = Boolean(held?.optional || field.optional);189 current.fields.set(fieldID, optional ? { id: fieldID, name, kind: field.kind, optional: true } : { id: fieldID, name, kind: field.kind });190 }191 types.set(typeID, current);192 }193 }194 return {195 types: [...types.values()].sort(byID).map((type) => ({196 id: type.id, name: type.name, fields: [...type.fields.values()].sort(byID),197 })),198 };199}200201export function projectFields(table: PackedTable, knownSchema: Schema, fileSchema: Schema): FieldProjection {202 const fileType = fileSchema.types.find((type) => normalizeID(type.id) === table.rootType);203 if (!fileType) throw new Error(`file schema does not describe table type ${table.rootType}`);204 const fileKinds = new Map(fileType.fields.map((field) => [normalizeID(field.id), field.kind]));205 const knownType = knownSchema.types.find((type) => normalizeID(type.id) === table.rootType);206 const knownKinds = new Map((knownType?.fields ?? []).map((field) => [normalizeID(field.id), field.kind]));207 const known: string[] = [];208 const foreign: string[] = [];209 for (const column of table.columns) {210 if (fileKinds.get(column.id) !== column.kind) throw new Error(`file schema disagrees with wire kind for ${column.id}`);211 const expected = knownKinds.get(column.id);212 if (expected === undefined) foreign.push(column.id);213 else if (expected !== column.kind) throw new Error(`known schema disagrees with wire kind for ${column.id}`);214 else known.push(column.id);215 }216 return { known, foreign };217}218219function boundedSlice(bytes: Uint8Array, offset64: bigint, length64: bigint): Uint8Array {220 if (offset64 > BigInt(Number.MAX_SAFE_INTEGER) || length64 > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error("column range exceeds JavaScript precision");221 const offset = Number(offset64);222 const length = Number(length64);223 if (offset > bytes.byteLength || length > bytes.byteLength - offset) throw new Error("column range exceeds block");224 return bytes.subarray(offset, offset + length);225}226227function normalizeID(id: string): string {228 const plain = id.toLowerCase().replaceAll("-", "");229 if (!/^[0-9a-f]{32}$/.test(plain)) throw new Error(`ID ${id} is not 128 bits`);230 return `${plain.slice(0, 8)}-${plain.slice(8, 12)}-${plain.slice(12, 16)}-${plain.slice(16, 20)}-${plain.slice(20)}`;231}232233function formatID(bytes: Uint8Array): string { return normalizeID(hex(bytes)); }234function hex(bytes: Uint8Array): string { return [...bytes].map((byte) => byte.toString(16).padStart(2, "0")).join(""); }235function byID(left: { readonly id: string }, right: { readonly id: string }): number { return left.id.localeCompare(right.id); }236237function crc32(...parts: readonly Uint8Array[]): number {238 let crc = 0xffffffff;239 for (const part of parts) {240 for (const byte of part) {241 crc ^= byte;242 for (let bit = 0; bit < 8; bit++) crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1));243 }244 }245 return (crc ^ 0xffffffff) >>> 0;246}