feat: rewrite csv parsing

config.default.json

@@ -1,5 +1,6 @@
 {
 	"aowowUrl": "https://wotlkdb.com",
+	"loadDbcs": true,
 	"realms": [
 		{
 			"name": "AzerothCore",

package-lock.json

@@ -1,18 +1,17 @@
 {
 	"name": "azerothcore-armory",
-	"version": "0.0.2",
+	"version": "0.1.0",
 	"lockfileVersion": 2,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "azerothcore-armory",
-			"version": "0.0.2",
+			"version": "0.1.0",
 			"license": "MIT",
 			"dependencies": {
 				"@supercharge/promise-pool": "^2.1.0",
 				"camelcase": "^6.2.1",
 				"cli-progress": "^3.9.1",
-				"csv-parser": "^3.0.0",
 				"express": "^4.17.2",
 				"express-handlebars": "^6.0.2",
 				"glob": "^7.2.0",
@@ -1226,20 +1225,6 @@
 				"node": ">=8"
 			}
 		},
-		"node_modules/csv-parser": {
-			"version": "3.0.0",
-			"resolved": "https://registry.npmjs.org/csv-parser/-/csv-parser-3.0.0.tgz",
-			"integrity": "sha512-s6OYSXAK3IdKqYO33y09jhypG/bSDHPuyCme/IdEHfWpLf/jKcpitVFyOC6UemgGk8v7Q5u2XE0vvwmanxhGlQ==",
-			"dependencies": {
-				"minimist": "^1.2.0"
-			},
-			"bin": {
-				"csv-parser": "bin/csv-parser"
-			},
-			"engines": {
-				"node": ">= 10"
-			}
-		},
 		"node_modules/date-fns": {
 			"version": "2.28.0",
 			"resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.28.0.tgz",
@@ -5679,14 +5664,6 @@
 			"integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==",
 			"dev": true
 		},
-		"csv-parser": {
-			"version": "3.0.0",
-			"resolved": "https://registry.npmjs.org/csv-parser/-/csv-parser-3.0.0.tgz",
-			"integrity": "sha512-s6OYSXAK3IdKqYO33y09jhypG/bSDHPuyCme/IdEHfWpLf/jKcpitVFyOC6UemgGk8v7Q5u2XE0vvwmanxhGlQ==",
-			"requires": {
-				"minimist": "^1.2.0"
-			}
-		},
 		"date-fns": {
 			"version": "2.28.0",
 			"resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.28.0.tgz",

package.json

@@ -1,14 +1,15 @@
 {
 	"name": "azerothcore-armory",
-	"version": "0.0.3",
+	"version": "0.1.0",
 	"description": "",
 	"scripts": {
 		"build": "tsc -p tsconfig.json",
 		"clean": "rimraf build/",
-		"start": "node build/armory/main.js",
+		"start": "node --expose-gc build/armory/main.js",
 		"watch": "concurrently \"tsc -w --project .\" \"npm run nodemon\"",
 		"nodemon": "nodemon -q -w build -w config.json build/armory/main.js",
-		"fetchdata": "node build/tools/fetchdata.js"
+		"fetchdata": "node build/tools/fetchdata.js",
+		"cleardata": "rimraf data/bone data/meta data/mo3 data/textures"
 	},
 	"author": "https://github.com/r-o-b-o-t-o",
 	"license": "MIT",
@@ -29,7 +30,6 @@
 		"@supercharge/promise-pool": "^2.1.0",
 		"camelcase": "^6.2.1",
 		"cli-progress": "^3.9.1",
-		"csv-parser": "^3.0.0",
 		"express": "^4.17.2",
 		"express-handlebars": "^6.0.2",
 		"glob": "^7.2.0",

src/armory/Armory.ts

@@ -5,21 +5,21 @@ import { Connection, createConnection } from "mysql2/promise";
 import { engine as handlebarsEngine } from "express-handlebars";
 
 import { Config } from "./Config";
-import { DbcReader } from "./data/DbcReader";
+import { DbcManager } from "./data/DbcReader";
 import { CharacterCustomization } from "./data/CharacterCustomization";
 import { IndexController } from "./controllers/IndexController";
 import { CharacterController } from "./controllers/CharacterController";
 
 export class Armory {
 	public characterCustomization: CharacterCustomization;
-	public dbcReader: DbcReader;
+	public dbc: DbcManager;
 	public config: Config;
 	public worldDb: Connection;
 
 	private charsDbs: { [key: string]: Connection };
 
 	public constructor() {
-		this.dbcReader = new DbcReader();
+		this.dbc = new DbcManager();
 		this.characterCustomization = new CharacterCustomization();
 		this.charsDbs = {};
 	}
@@ -31,7 +31,9 @@ export class Armory {
 		console.log("Loading config...");
 		this.config = await Config.load();
 		console.log("Loading data files...");
-		await this.dbcReader.loadAllFiles();
+		if (this.config.loadDbcs) {
+			await this.dbc.loadAllFiles();
+		}
 		await this.characterCustomization.loadData();
 
 		console.log("Connecting to databases...");
@@ -68,6 +70,7 @@ export class Armory {
 		await charsController.load();
 		app.get("/character/:realm/:name", charsController.character.bind(charsController));
 
+		this.gc();
 		app.listen(listenPort, "0.0.0.0", () => {
 			console.log(`Server is listening on 0.0.0.0:${listenPort}.`);
 		});
@@ -76,4 +79,16 @@ export class Armory {
 	public getCharactersDb(realm: string): Connection {
 		return this.charsDbs[realm.toLowerCase()];
 	}
+
+	public gc(): void {
+		if (this.config.loadDbcs) {
+			return;
+		}
+
+		setTimeout(() => {
+			if (global.gc) {
+				global.gc();
+			}
+		}, 500);
+	}
 }

src/armory/Config.ts

@@ -16,6 +16,7 @@ export interface IRealmConfig {
 
 export class Config {
 	public aowowUrl: string;
+	public loadDbcs: boolean;
 	public realms: IRealmConfig[];
 	public worldDatabase: IDatabaseConfig;
 

src/armory/controllers/CharacterController.ts

@@ -47,8 +47,9 @@ export class CharacterController {
 
 	public async load(): Promise<void> {
 		this.itemInventoryTypes = {};
-		for (const item of this.armory.dbcReader.dbcItem) {
+		const itemsRetail = await this.armory.dbc.itemRetail().toArray();
-			const retailItem = this.armory.dbcReader.dbcItemRetail.find(row => row.id === item.id);
+		for await (const item of this.armory.dbc.item()) {
+			const retailItem = itemsRetail.find(row => row.id === item.id);
 			if (retailItem !== undefined) {
 				this.itemInventoryTypes[item.id] = retailItem.inventoryType;
 			}
@@ -56,10 +57,10 @@ export class CharacterController {
 
 		this.itemIcons = {};
 		const itemIconsByDisplayInfoId: { [key: number]: number } = {};
-		for (const row of this.armory.dbcReader.dbcItemDisplayInfo) {
+		for await (const row of this.armory.dbc.itemDisplayInfo()) {
 			itemIconsByDisplayInfoId[row.id] = row.inventoryIcon0;
 		}
-		for (const item of this.armory.dbcReader.dbcItem) {
+		for await (const item of this.armory.dbc.item()) {
 			const icon = itemIconsByDisplayInfoId[item.displayInfoId];
 			if (icon !== undefined) {
 				this.itemIcons[item.id] = icon;
@@ -67,12 +68,12 @@ export class CharacterController {
 		}
 
 		this.gemItems = {};
-		for (const row of this.armory.dbcReader.dbcItem.filter(item => item.classId === ITEM_CLASS_GEM)) {
+		for await (const row of this.armory.dbc.item().filter(item => item.classId === ITEM_CLASS_GEM)) {
 			this.gemItems[row.id] = true;
 		}
 
 		this.enchantSrcItems = {};
-		for (const row of this.armory.dbcReader.dbcSpellItemEnchantment) {
+		for await (const row of this.armory.dbc.spellItemEnchantment()) {
 			this.enchantSrcItems[row.id] = row.srcItemId;
 		}
 
@@ -100,7 +101,7 @@ export class CharacterController {
 			return;
 		}
 		const equipmentData = await this.getEquipmentData(realm, charData.guid);
-		const customization = await this.getCustomizationOptions(charData);
+		const customization = this.getCustomizationOptions(charData);
 		const equipment = equipmentData.map(row => {
 			(row as any).icon = this.itemIcons[row.itemEntry];
 			(row as any).gems = this.getGemsFromEnchantments(row.enchantments);
@@ -117,11 +118,13 @@ export class CharacterController {
 				gender: charData.gender,
 				level: charData.level,
 				online: charData.online === 1,
-				characterModelItems: this.getModelViewerItems(equipmentData, charData.class),
+				characterModelItems: await this.getModelViewerItems(equipmentData, charData.class),
 				customizationOptions: customization,
 				equipment,
 			}),
 		});
+
+		this.armory.gc();
 	}
 
 	private async getCharacterData(realm: string, charName: string): Promise<ICharacterData> {
@@ -147,7 +150,7 @@ export class CharacterController {
 		return rows as RowDataPacket[] as IEquipmentData[];
 	}
 
-	private getModelViewerItems(equipmentData: IEquipmentData[], charClass: number): number[][] {
+	private async getModelViewerItems(equipmentData: IEquipmentData[], charClass: number): Promise<number[][]> {
 		if (charClass !== 3) {
 			// Keep ranged weapon only if the character is a hunter
 			equipmentData = equipmentData.filter(row => row.slot !== 17);
@@ -160,11 +163,11 @@ export class CharacterController {
 
 		const items: number[][] = [];
 		for (const equipment of visibleEquipment) {
-			const modifiedAppearance = this.armory.dbcReader.dbcItemModifiedAppearance.find(row => row.itemId === equipment.itemEntry);
+			const modifiedAppearance = await this.armory.dbc.itemModifiedAppearance().find(row => row.itemId === equipment.itemEntry);
 			if (modifiedAppearance === undefined) {
 				continue;
 			}
-			const appearance = this.armory.dbcReader.dbcItemAppearance.find(row => row.id === modifiedAppearance.itemAppearanceId);
+			const appearance = await this.armory.dbc.itemAppearance().find(row => row.id === modifiedAppearance.itemAppearanceId);
 			if (appearance === undefined) {
 				continue;
 			}
@@ -195,7 +198,7 @@ export class CharacterController {
 			.filter(enchant => enchant in this.enchantSrcItems && !(this.enchantSrcItems[enchant] in this.gemItems) && enchant !== socketBonus);
 	}
 
-	private async getCustomizationOptions(charData: ICharacterData): Promise<ICustomizationOption[]> {
+	private getCustomizationOptions(charData: ICharacterData): ICustomizationOption[] {
 		const data = this.armory.characterCustomization.getCharacterCustomizationData(charData.race, charData.gender);
 		const options = [];
 		const setOptionByChoiceIndex = (optionName: string, choiceIndex: number) => {

src/armory/data/DbcReader.ts

@@ -1,50 +1,29 @@
 import * as fs from "fs";
 import * as path from "path";
 
-import * as csv from "csv-parser";
 import * as camelCase from "camelcase";
 
 export interface IItemDbc {
 	id: number;
 	classId: number;
-	subclassId: number;
-	soundOverrideSubclassId: number;
-	material: number;
 	displayInfoId: number;
 	inventoryType: number;
-	sheatheType: number;
 }
 
 export interface IItemRetailDbc {
 	id: number;
-	classId: number;
-	subclassId: number;
-	material: number;
 	inventoryType: number;
-	sheatheType: number;
-	soundOverrideSubclassId: number;
-	iconFileDataId: number;
-	itemGroupSoundsId: number;
-	contentTuningId: number;
-	modifiedCraftingReagentItemId: number;
 }
 
 export interface IItemAppearanceDbc {
 	id: number;
-	displayType: number;
 	itemDisplayInfoId: number;
-	defaultIconFileDataId: number;
-	uiOrder: number;
-	playerConditionId: number;
 }
 
 export interface IItemModifiedAppearanceDbc {
 	id: number;
 	itemId: number;
-	itemAppearanceModifierId: number;
 	itemAppearanceId: number;
-	orderIndex: number;
-	transmogSourceTypeEnum: number;
 }
 
 export interface IItemDisplayInfoDbc {
@@ -53,25 +32,13 @@ export interface IItemDisplayInfoDbc {
 }
 
 export interface IMountDbc {
-	nameLang: string;
-	sourceTextLang: string;
-	descriptionLang: string;
 	id: number;
-	mountTypeId: number;
-	flags: number;
-	sourceTypeEnum: number;
 	sourceSpellId: number;
-	playerConditionId: number;
-	mountFlyRideHeight: number;
-	uiModelSceneId: number;
-	mountSpecialRiderAnimKitId: number;
-	mountSpecialSpellVisualKitId: number;
 }
 
 export interface IMountXDisplayDbc {
 	id: number;
 	creatureDisplayInfoId: number;
-	playerConditionId: number;
 	mountId: number;
 }
 
@@ -85,45 +52,300 @@ export interface ISpellItemEnchantmentDbc {
 	srcItemId: number;
 }
 
-export class DbcReader {
+interface IAsyncGeneratorWithArrayMethods<T> {
-	public dbcItem: IItemDbc[];
+	[Symbol.asyncIterator](): AsyncGenerator<T>;
-	public dbcItemRetail: IItemRetailDbc[];
+	toArray(): Promise<T[]>;
-	public dbcItemAppearance: IItemAppearanceDbc[];
+	map<M>(fn: (t: T) => M): IAsyncGeneratorWithArrayMethods<M>;
-	public dbcItemModifiedAppearance: IItemModifiedAppearanceDbc[];
+	filter(fn: (t: T) => boolean): IAsyncGeneratorWithArrayMethods<T>;
-	public dbcItemDisplayInfo: IItemDisplayInfoDbc[];
+	find(fn: (t: T) => boolean): Promise<T>;
-	public dbcMount: IMountDbc[];
+}
-	public dbcMountDisplay: IMountXDisplayDbc[];
-	public dbcSpell: ISpellDbc[];
-	public dbcSpellItemEnchantment: ISpellItemEnchantmentDbc[];
 
-	public readDbcFile<T>(file: string): Promise<T[]> {
+class ArrayAsAsyncGenerator<T> implements IAsyncGeneratorWithArrayMethods<T> {
-		return new Promise((res, rej) => {
+	private data: T[];
-			const rows = [];
 
-			fs.createReadStream(file)
+	public constructor(data: T[]) {
-				.pipe(csv({
+		this.data = data;
-					mapHeaders: ({ header, index }) => camelCase(header).replace(/[\[\]]/g, ""),
-					mapValues: ({ header, index, value }) => isNaN(value) ? value : Number(value),
-				}))
-				.on("error", rej)
-				.on("data", (data) => rows.push(data))
-				.on("end", () => {
-					res(rows);
-				});
-		});
 	}
 
-	public async loadAllFiles(): Promise<void> {
+	async toArray(): Promise<T[]> {
-		const dir = path.join(process.cwd(), "data");
+		return this.data;
+	}
 
-		this.dbcItem = await this.readDbcFile<IItemDbc>(path.join(dir, "Item_3.3.5_12340.csv"));
+	async *[Symbol.asyncIterator](): AsyncGenerator<T> {
-		this.dbcItemRetail = await this.readDbcFile<IItemRetailDbc>(path.join(dir, "Item_9.2.0_41462.csv"));
+		for (const x of this.data) {
-		this.dbcItemAppearance = await this.readDbcFile<IItemAppearanceDbc>(path.join(dir, "ItemAppearance_9.2.0_41462.csv"));
+			yield x;
-		this.dbcItemModifiedAppearance = await this.readDbcFile<IItemModifiedAppearanceDbc>(path.join(dir, "ItemModifiedAppearance_9.2.0_41462.csv"));
+		}
-		this.dbcItemDisplayInfo = await this.readDbcFile<IItemDisplayInfoDbc>(path.join(dir, "ItemDisplayInfo_3.3.5_12340.csv"));
+	}
-		this.dbcMount = await this.readDbcFile<IMountDbc>(path.join(dir, "Mount_9.2.0_41462.csv"));
+
-		this.dbcMountDisplay = await this.readDbcFile<IMountXDisplayDbc>(path.join(dir, "MountXDisplay_9.2.0_41462.csv"));
+	public map<M>(fn: (t: T) => M): ArrayAsAsyncGenerator<M> {
-		this.dbcSpell = await this.readDbcFile<ISpellDbc>(path.join(dir, "Spell_3.3.5_12340.csv"));
+		return new ArrayAsAsyncGenerator<M>(this.data.map(fn));
-		this.dbcSpellItemEnchantment = await this.readDbcFile<ISpellItemEnchantmentDbc>(path.join(dir, "SpellItemEnchantment_3.3.5_12340.csv"));
+	}
+
+	filter(fn: (t: T) => boolean): ArrayAsAsyncGenerator<T> {
+		return new ArrayAsAsyncGenerator<T>(this.data.filter(fn));
+	}
+
+	async find(fn: (t: T) => boolean): Promise<T> {
+		return this.data.find(fn);
+	}
+}
+
+class AsyncGenWrapper<T> implements IAsyncGeneratorWithArrayMethods<T> {
+	private gen: AsyncGenerator<T>;
+
+	public constructor(gen: AsyncGenerator<T>) {
+		this.gen = gen;
+	}
+
+	public static from<T>(array: T[]): AsyncGenWrapper<T> {
+		return new AsyncGenWrapper<T>(async function* () {
+			for (const x of array) {
+				yield x;
+			}
+		}());
+	}
+
+	public async *[Symbol.asyncIterator](): AsyncGenerator<T> {
+		for await (const x of this.gen) {
+			yield x;
+		}
+	}
+
+	public async toArray(): Promise<T[]> {
+		const values = [];
+		for await (const x of this) {
+			values.push(x);
+		}
+		return values;
+	}
+
+	private wrap<X>(g: (that: AsyncGenWrapper<T>) => AsyncGenerator<X>): AsyncGenWrapper<X> {
+		return new AsyncGenWrapper<X>(g(this));
+	}
+
+	public map<M>(fn: (t: T) => M): AsyncGenWrapper<M> {
+		return this.wrap(async function* (me) {
+			for await (const x of me) {
+				yield fn(x);
+			}
+		});
+	}
+
+	public filter(fn: (t: T) => boolean): AsyncGenWrapper<T> {
+		return this.wrap(async function* (me) {
+			for await (const x of me) {
+				if (fn(x)) {
+					yield x;
+				}
+			}
+		});
+	}
+
+	public async find(fn: (t: T) => boolean): Promise<T> {
+		for await (const x of this) {
+			if (fn(x)) {
+				return x;
+			}
+		}
+	}
+}
+
+class DbcReader<T> {
+	private filePath: string;
+	private fields: string[];
+
+	public constructor(filePath: string, keepFields: string[] = []) {
+		this.filePath = filePath;
+		this.fields = keepFields;
+	}
+
+	public async *read(): AsyncGenerator<T> {
+		const stream = fs.createReadStream(this.filePath);
+		const itr = this.readLines(stream);
+		const headerLine = await itr.next();
+		if (headerLine.done === true) {
+			return;
+		}
+
+		const headerCols = this.parseCsvLine(headerLine.value)
+			.map(header => camelCase(header).replace(/[\[\]]/g, ""));
+
+		for await (const line of itr) {
+			const cols = this.parseCsvLine(line)
+				.map(value => {
+					const parsed: number = parseInt(value, 10);
+					return isNaN(parsed) ? value : parsed;
+				});
+			const row = {};
+			headerCols.forEach((header, headerIdx) => {
+				if (this.fields.length === 0 || this.fields.includes(header)) {
+					row[header] = cols[headerIdx];
+				}
+			});
+			yield row as T;
+		}
+	}
+
+	private async *readLines(stream: fs.ReadStream): AsyncGenerator<string> {
+		let previous = "";
+
+		for await (const chunk of stream) {
+			previous += chunk;
+			let unixEolIndex: number;
+			let winEolIndex: number;
+
+			while ((unixEolIndex = previous.indexOf("\n")) >= 0 || (winEolIndex = previous.indexOf("\r\n")) >= 0) {
+				const unix = unixEolIndex >= 0;
+				const line = previous.slice(0, (unix ? unixEolIndex : winEolIndex) - 1);
+				yield line;
+				previous = previous.slice(unix ? (unixEolIndex + 1) : (winEolIndex + 2));
+			}
+		}
+
+		if (previous.length > 0) {
+			yield previous;
+		}
+	}
+
+	private parseCsvLine(line: string): string[] {
+		// Adapted from https://stackoverflow.com/a/14991797
+		const arr = [];
+		let quote = false; // 'true' means we're inside a quoted field
+
+		// Iterate over each character, keep track of current row and column (of the returned array)
+		for (let col = 0, c = 0; c < line.length; c++) {
+			let cc = line[c], nc = line[c + 1]; // Current character, next character
+			arr[col] = arr[col] || ""; // Create a new column (start with empty string) if necessary
+
+			// If the current character is a quotation mark, and we're inside a
+			// quoted field, and the next character is also a quotation mark,
+			// add a quotation mark to the current column and skip the next character
+			if (cc == '"' && quote && nc == '"') {
+				arr[col] += cc; ++c;
+				continue;
+			}
+
+			// If it's just one quotation mark, begin/end quoted field
+			if (cc == '"') {
+				quote = !quote;
+				continue;
+			}
+
+			// If it's a comma and we're not in a quoted field, move on to the next column
+			if (cc == ',' && !quote) {
+				++col;
+				continue;
+			}
+
+			// If it's a newline (CRLF) and we're not in a quoted field, move on to the next row
+			if (cc == '\r' && nc == '\n' && !quote) {
+				break;
+			}
+
+			// If it's a newline (LF or CR) and we're not in a quoted field, move on to the next row
+			if ((cc == '\n' && !quote) ||
+				(cc == '\r' && !quote)) {
+				break;
+			}
+
+			// Otherwise, append the current character to the current column
+			arr[col] += cc;
+		}
+
+		return arr;
+	}
+}
+
+const dir = path.join(process.cwd(), "data");
+export const DbcFiles = {
+	item: path.join(dir, "Item_3.3.5_12340.csv"),
+	itemRetail: path.join(dir, "Item_9.2.0_41462.csv"),
+	itemAppearance: path.join(dir, "ItemAppearance_9.2.0_41462.csv"),
+	itemModifiedAppearance: path.join(dir, "ItemModifiedAppearance_9.2.0_41462.csv"),
+	itemDisplayInfo: path.join(dir, "ItemDisplayInfo_3.3.5_12340.csv"),
+	mount: path.join(dir, "Mount_9.2.0_41462.csv"),
+	mountDisplay: path.join(dir, "MountXDisplay_9.2.0_41462.csv"),
+	spell: path.join(dir, "Spell_3.3.5_12340.csv"),
+	spellItemEnchantment: path.join(dir, "SpellItemEnchantment_3.3.5_12340.csv"),
+};
+
+const dbcFields = {
+	item: ["id", "classId", "displayInfoId", "inventoryType"],
+	itemRetail: ["id", "inventoryType"],
+	itemAppearance: ["id", "itemDisplayInfoId"],
+	itemModifiedAppearance: ["id", "itemId", "itemAppearanceId"],
+	itemDisplayInfo: ["id", "inventoryIcon0"],
+	mount: ["id", "sourceSpellId"],
+	mountDisplay: ["id", "creatureDisplayInfoId", "mountId"],
+	spell: ["id", "mechanic"],
+	spellItemEnchantment: ["id", "srcItemId"],
+};
+
+export class DbcManager {
+	private _item: IItemDbc[];
+	private _itemRetail: IItemRetailDbc[];
+	private _itemAppearance: IItemAppearanceDbc[];
+	private _itemModifiedAppearance: IItemModifiedAppearanceDbc[];
+	private _itemDisplayInfo: IItemDisplayInfoDbc[];
+	private _mount: IMountDbc[];
+	private _mountDisplay: IMountXDisplayDbc[];
+	private _spell: ISpellDbc[];
+	private _spellItemEnchantment: ISpellItemEnchantmentDbc[];
+
+	public async loadAllFiles(): Promise<void> {
+		this._item = await this.read<IItemDbc>(DbcFiles.item, dbcFields.item).toArray();
+		this._itemRetail = await this.read<IItemRetailDbc>(DbcFiles.itemRetail, dbcFields.itemRetail).toArray();
+		this._itemAppearance = await this.read<IItemAppearanceDbc>(DbcFiles.itemAppearance, dbcFields.itemAppearance).toArray();
+		this._itemModifiedAppearance = await this.read<IItemModifiedAppearanceDbc>(DbcFiles.itemModifiedAppearance, dbcFields.itemModifiedAppearance).toArray();
+		this._itemDisplayInfo = await this.read<IItemDisplayInfoDbc>(DbcFiles.itemDisplayInfo, dbcFields.itemDisplayInfo).toArray();
+		this._mount = await this.read<IMountDbc>(DbcFiles.mount, dbcFields.mount).toArray();
+		this._mountDisplay = await this.read<IMountXDisplayDbc>(DbcFiles.mountDisplay, dbcFields.mountDisplay).toArray();
+		this._spell = await this.read<ISpellDbc>(DbcFiles.spell, dbcFields.spell).toArray();
+		this._spellItemEnchantment = await this.read<ISpellItemEnchantmentDbc>(DbcFiles.spellItemEnchantment, dbcFields.spellItemEnchantment).toArray();
+	}
+
+	public item() {
+		return this.getLoadedDataOrRead(DbcFiles.item, this._item, dbcFields.item);
+	}
+
+	public itemRetail() {
+		return this.getLoadedDataOrRead(DbcFiles.itemRetail, this._itemRetail, dbcFields.itemRetail);
+	}
+
+	public itemAppearance() {
+		return this.getLoadedDataOrRead(DbcFiles.itemAppearance, this._itemAppearance, dbcFields.itemAppearance);
+	}
+
+	public itemModifiedAppearance() {
+		return this.getLoadedDataOrRead(DbcFiles.itemModifiedAppearance, this._itemModifiedAppearance, dbcFields.itemModifiedAppearance);
+	}
+
+	public itemDisplayInfo() {
+		return this.getLoadedDataOrRead(DbcFiles.itemDisplayInfo, this._itemDisplayInfo, dbcFields.itemDisplayInfo);
+	}
+
+	public mount() {
+		return this.getLoadedDataOrRead(DbcFiles.mount, this._mount, dbcFields.mount);
+	}
+
+	public mountDisplay() {
+		return this.getLoadedDataOrRead(DbcFiles.mountDisplay, this._mountDisplay, dbcFields.mountDisplay);
+	}
+
+	public spell() {
+		return this.getLoadedDataOrRead(DbcFiles.spell, this._spell, dbcFields.spell);
+	}
+
+	public spellItemEnchantment() {
+		return this.getLoadedDataOrRead(DbcFiles.spellItemEnchantment, this._spellItemEnchantment, dbcFields.spellItemEnchantment);
+	}
+
+	private read<T>(file: string, keepFields: string[] = []): AsyncGenWrapper<T> {
+		const reader = new DbcReader<T>(file, keepFields);
+		return new AsyncGenWrapper(reader.read());
+	}
+
+	private getLoadedDataOrRead<T>(path: string, data: T[], keepFields: string[] = []): IAsyncGeneratorWithArrayMethods<T> {
+		return data === undefined ? this.read<T>(path, keepFields) : new ArrayAsAsyncGenerator(data);
 	}
 }

src/tools/fetchdata.ts

@@ -11,7 +11,7 @@ import * as prettyMs from "pretty-ms";
 import * as cliProgress from "cli-progress";
 import promisepool = require("@supercharge/promise-pool");
 
-import { DbcReader, IItemAppearanceDbc, IItemModifiedAppearanceDbc, IMountDbc, IMountXDisplayDbc } from "../armory/data/DbcReader";
+import { DbcManager, IItemAppearanceDbc, IItemModifiedAppearanceDbc, IMountDbc, IMountXDisplayDbc } from "../armory/data/DbcReader";
 
 require("source-map-support").install();
 
@@ -73,7 +73,7 @@ class HttpRequestError extends Error {
 	}
 }
 
-let dbcReader: DbcReader;
+let dbc: DbcManager;
 let dbcItemAppearanceById: { [key: number]: IItemAppearanceDbc };
 let dbcItemModifiedAppearanceByItemId: { [key: number]: IItemModifiedAppearanceDbc };
 let dbcMountBySourceSpellId: { [key: number]: IMountDbc };
@@ -215,7 +215,7 @@ async function downloadRaces(): Promise<void> {
 }
 
 async function downloadArmors(): Promise<void> {
-	const rows = dbcReader.dbcItem.filter((row) => row.classId === classIdArmor);
+	const rows = await dbc.item().filter((row) => row.classId === classIdArmor).toArray();
 
 	const progress = new Progress("Downloading armor data...", rows.length);
 
@@ -247,7 +247,7 @@ async function downloadArmors(): Promise<void> {
 }
 
 async function downloadWeapons(): Promise<void> {
-	const rows = dbcReader.dbcItem.filter((row) => row.classId === classIdWeapon);
+	const rows = await dbc.item().filter((row) => row.classId === classIdWeapon).toArray();
 
 	const progress = new Progress("Downloading weapon data...", rows.length);
 
@@ -280,32 +280,32 @@ async function downloadWeapons(): Promise<void> {
 async function readDbcData(): Promise<void> {
 	console.log("Reading DBC data...");
 
-	dbcReader = new DbcReader();
+	dbc = new DbcManager();
-	await dbcReader.loadAllFiles();
+	await dbc.loadAllFiles();
 
 	dbcItemAppearanceById = {};
-	for (const row of dbcReader.dbcItemAppearance) {
+	for await (const row of dbc.itemAppearance()) {
 		dbcItemAppearanceById[row.id] = row;
 	}
 
 	dbcItemModifiedAppearanceByItemId = {};
-	for (const row of dbcReader.dbcItemModifiedAppearance) {
+	for await (const row of dbc.itemModifiedAppearance()) {
 		dbcItemModifiedAppearanceByItemId[row.itemId] = row;
 	}
 
 	dbcMountBySourceSpellId = {};
-	for (const row of dbcReader.dbcMount) {
+	for await (const row of dbc.mount()) {
 		dbcMountBySourceSpellId[row.sourceSpellId] = row;
 	}
 
 	dbcMountDisplayByMountId = {};
-	for (const row of dbcReader.dbcMountDisplay) {
+	for await (const row of dbc.mountDisplay()) {
 		dbcMountDisplayByMountId[row.mountId] = row;
 	}
 }
 
 async function downloadMounts(): Promise<void> {
-	const mountSpells = dbcReader.dbcSpell.filter(spell => spell.mechanic === spellMechanicMounted);
+	const mountSpells = await dbc.spell().filter(spell => spell.mechanic === spellMechanicMounted).toArray();
 	const progress = new Progress("Downloading mount data...", mountSpells.length);
 
 	await promisepool.PromisePool