Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@
"packageManager": "pnpm@7.17.0",
"dependencies": {
"bson": "^4.7.0",
"lodash.clonedeep": "^4.5.0"
"lodash.clonedeep": "^4.5.0",
"lz-string": "^1.4.4"
},
"devDependencies": {
"@commitlint/cli": "^17.3.0",
Expand All @@ -68,6 +69,7 @@
"@swc/jest": "^0.2.23",
"@types/jest": "^29.2.3",
"@types/lodash.clonedeep": "^4.5.7",
"@types/lz-string": "^1.3.34",
"@typescript-eslint/eslint-plugin": "^5.44.0",
"@typescript-eslint/parser": "^5.44.0",
"concurrently": "^7.6.0",
Expand Down
19 changes: 19 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

31 changes: 29 additions & 2 deletions src/stringify.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { EJSON } from 'bson';
import cloneDeep from 'lodash.clonedeep';
import { compressToUTF16 } from 'lz-string';
import {
MINIFY_REMAINING_CANDIDATES,
MINIFY_STARTING_CANDIDATES,
Expand All @@ -10,16 +11,19 @@ export type StringifyReplacer = (this: any, key: string, value: any) => any;
export interface StringifyOptions {
extended?: boolean | { enable: boolean; relaxed?: boolean };
minify?: false | { whitespace?: boolean; key?: boolean };
compress?: boolean | { enable: boolean };
}

interface _StringifyOptions {
extended: { enable: boolean; relaxed: boolean };
minify: { whitespace: boolean; key: boolean };
compress: { enable: boolean };
}

const defaultOptions: _StringifyOptions = {
extended: { enable: false, relaxed: true },
minify: { whitespace: false, key: false },
compress: { enable: false },
};

function mergeWithDefaultOptions(input?: StringifyOptions): _StringifyOptions {
Expand Down Expand Up @@ -53,6 +57,18 @@ function mergeWithDefaultOptions(input?: StringifyOptions): _StringifyOptions {
};
}

if (input.compress == null) {
input.compress = defaultOptions.compress;
} else if (typeof input.compress === 'boolean') {
input.compress = {
enable: input.compress,
};
} else {
input.compress = {
enable: input.compress.enable,
};
}

return input as _StringifyOptions;
}

Expand Down Expand Up @@ -80,12 +96,19 @@ export function stringify(
obj = minifyJsonKeys(obj);
}

let result: string;
if (_options.extended.enable) {
return EJSON.stringify(obj, _replacer, space, {
result = EJSON.stringify(obj, _replacer, space, {
relaxed: _options.extended.relaxed,
});
} else {
result = JSON.stringify(obj, _replacer, space);
}

if (_options.compress.enable) {
return compressString(result);
}
return JSON.stringify(obj, _replacer, space);
return result;
}

export interface KeyMinifiedJson<T> {
Expand Down Expand Up @@ -175,3 +198,7 @@ function minifyAllKeys(obj: any, reverseKeyMap: Record<string, string>): any {

return obj;
}

export function compressString(str: string): string {
return compressToUTF16(str);
}
81 changes: 52 additions & 29 deletions test/stringify.perf.test.ts
Original file line number Diff line number Diff line change
@@ -1,44 +1,67 @@
import { describe, expect, it } from '@jest/globals';
import { describe, it } from '@jest/globals';
import { JsonKit } from '../src';
import { formatDuration, timer } from './util';

describe('[stringify] performance', () => {
it('stringify with ~650KB json data', async () => {
const data = await import('./dataset/ne_110m_populated_places.json');
const test = (data: any): void => {
const [original, originalDuration] = timer(() => JSON.stringify(data));

const original = JSON.stringify(data);
const [basic, basicDuration] = timer(() => JsonKit.stringify(data));

const basic = JsonKit.stringify(data);
const [minified, minifiedDuration] = timer(() =>
JsonKit.stringify(data, { minify: { key: true } })
);

const minified = JsonKit.stringify(data, { minify: { key: true } });
const [compressed, compressedDuration] = timer(() =>
JsonKit.stringify(data, { compress: true })
);

expect(basic).toEqual(original);
const [minifiedAndCompressed, minifiedAndCompressedDuration] = timer(() =>
JsonKit.stringify(data, {
minify: { key: true },
compress: true,
})
);

console.info(
`${original.length} -> ${minified.length} = ${(
((minified.length - original.length) / original.length) *
100
).toFixed(2)}%`
);
expect(minified.length).toBeLessThan(original.length);
});
console.info(
`baseline: ${original.length} (±0.00%) [${formatDuration(
originalDuration
)}]`
);

it('stringify with ~50MB json data', async () => {
const data = await import('./dataset/ne_10m_roads.json');
console.info(
`basic: ${basic.length} (±0.00%) [${formatDuration(basicDuration)}]`
);

const original = JSON.stringify(data);
console.info(
`minify: ${minified.length} (${(
((minified.length - original.length) / original.length) *
100
).toFixed(2)}%) [${formatDuration(minifiedDuration)}]`
);

const basic = JsonKit.stringify(data);
console.info(
`compress: ${compressed.length} (${(
((compressed.length - original.length) / original.length) *
100
).toFixed(2)}%) [${formatDuration(compressedDuration)}]`
);

const minified = JsonKit.stringify(data, { minify: { key: true } });
console.info(
`minify + compress: ${minifiedAndCompressed.length} (${(
((minifiedAndCompressed.length - original.length) / original.length) *
100
).toFixed(2)}%) [${formatDuration(minifiedAndCompressedDuration)}]`
);
};

expect(basic).toEqual(original);
describe('[stringify] performance', () => {
it('stringify with ~650KB json data', async () => {
const data = await import('./dataset/ne_110m_populated_places.json');
test(data);
});

console.info(
`${original.length} -> ${minified.length} = ${(
((minified.length - original.length) / original.length) *
100
).toFixed(2)}%`
);
expect(minified.length).toBeLessThan(original.length);
it('stringify with ~50MB json data', async () => {
const data = await import('./dataset/ne_10m_roads.json');
test(data);
});
});
56 changes: 56 additions & 0 deletions test/stringify.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,18 +52,22 @@ describe('[stringify] basic', () => {
const optionsShorthand = JsonKit.stringify(obj, {
extended: true,
minify: false,
compress: true,
});
const optionsFull = JsonKit.stringify(obj, {
extended: { enable: false },
minify: { whitespace: false, key: false },
compress: { enable: false },
});
const optionsDefaultFull = JsonKit.stringify(obj, {
extended: { enable: true, relaxed: true },
minify: { whitespace: false, key: false },
compress: { enable: true },
});
const defaultOptions = JsonKit.stringify(obj, {
extended: { enable: false, relaxed: true },
minify: { whitespace: false, key: false },
compress: { enable: false },
});

expect(base).toEqual(defaultOptions);
Expand Down Expand Up @@ -167,3 +171,55 @@ describe('[stringify] minify', () => {
);
});
});

describe('[stringify] compress', () => {
const obj = {
a: 1,
'2': '2',
'': true,
' ': null,
'a-b': undefined,
_c: new Date(),
toString: () => 'string',
arr: [1, '2', true, null, undefined, new Date(), () => 'string', [], {}],
nested: {
a: 1,
'2': '2',
},
empty: {},
};

it('stringify with compression should have identical behavior as built-in JSON.stringify and compression', () => {
const replacer: StringifyReplacer = () => 'test';

expect(JsonKit.stringify(obj, { compress: true })).toEqual(
JsonKit.compressString(JSON.stringify(obj))
);

expect(JsonKit.stringify(obj, replacer, ' ', { compress: true })).toEqual(
JsonKit.compressString(JSON.stringify(obj, replacer, ' '))
);
});

it('stringify with compression should have identical behavior as bson.EJSON.stringify and compression', () => {
const replacer: StringifyReplacer = () => 'test';

expect(
JsonKit.stringify(obj, {
extended: { enable: true, relaxed: false },
compress: { enable: true },
})
).toEqual(JsonKit.compressString(EJSON.stringify(obj, { relaxed: false })));

expect(
JsonKit.stringify(obj, {
extended: { enable: true, relaxed: true },
compress: { enable: true },
})
).toEqual(JsonKit.compressString(EJSON.stringify(obj, { relaxed: true })));

expect(
JsonKit.stringify(obj, replacer, 4, { extended: true, compress: true })
).toEqual(JsonKit.compressString(EJSON.stringify(obj, replacer, 4)));
});
});
23 changes: 23 additions & 0 deletions test/util/format.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
export function formatDuration(milliseconds: number) {
if (milliseconds < 1000) {
return `${milliseconds.toFixed(0)} ms`;
}

const seconds = milliseconds / 1000;
if (seconds < 60) {
return `${seconds.toFixed(3)} s`;
}

const minutes = seconds / 60;
if (minutes < 60) {
return `${minutes.toFixed(2)} m`;
}

const hours = minutes / 60;
if (hours < 24) {
return `${hours.toFixed(2)} h`;
}

const days = hours / 24;
return `${days.toFixed(2)} d`;
}
2 changes: 2 additions & 0 deletions test/util/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
export * from './format';
export * from './timer';
27 changes: 27 additions & 0 deletions test/util/timer.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/**
* @param target target function to measure
* @returns Array of [result, durationInMs]
*/
export function timer<T>(target: () => T): [T, number];
export async function timer<T extends Promise<any>>(
target: () => T
): Promise<[Awaited<T>, number]>;
export function timer(
target: () => any
): [any, number] | Promise<[any, number]> {
const start = performance.now();
const result = target();
if (typeof result?.then === 'function') {
return new Promise((resolve, reject) => {
result
.then((awaited: any) => {
const durationInMs = performance.now() - start;
return resolve([awaited, durationInMs]);
})
.catch((error: any) => reject(error));
});
} else {
const durationInMs = performance.now() - start;
return [result, durationInMs];
}
}