diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e2cc02ee..eceb69bb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - TypeScript has been updated to 3.9. See [release notes](https://devblogs.microsoft.com/typescript/announcing-typescript-3-9/) for details. This update includes some fixes specific to our API usage: + - Importing a non-module using `import "./file"` produced a TS2307 error [#35973](https://github.com/microsoft/TypeScript/issues/35973) - TypeScript now tries to find a call signature even in presence of type errors (#36665)(https://github.com/microsoft/TypeScript/pull/36665): ```ts @@ -18,6 +19,12 @@ foo(1) ``` +- Reduced memory consumption and optimized performance of generators and iterators + +- Fixed generator syntax being ignored on methods (`*foo() {}`) and function expressions (`function*() {}`) + +- Fixed iteration over generators stopping at first yielded `nil` value + ## 0.33.0 - Added support for nullish coalescing `A ?? B`. diff --git a/src/LuaLib.ts b/src/LuaLib.ts index 8c27e5730..33437a7b8 100644 --- a/src/LuaLib.ts +++ b/src/LuaLib.ts @@ -33,6 +33,7 @@ export enum LuaLibFeature { FunctionApply = "FunctionApply", FunctionBind = "FunctionBind", FunctionCall = "FunctionCall", + Generator = "Generator", InstanceOf = "InstanceOf", InstanceOfObject = "InstanceOfObject", Iterator = "Iterator", @@ -72,6 +73,7 @@ const luaLibDependencies: Partial> = { ArrayFlat: [LuaLibFeature.ArrayConcat], ArrayFlatMap: [LuaLibFeature.ArrayConcat], Error: [LuaLibFeature.New, LuaLibFeature.Class, LuaLibFeature.FunctionCall], + Generator: [LuaLibFeature.Symbol], InstanceOf: [LuaLibFeature.Symbol], Iterator: [LuaLibFeature.Symbol], ObjectFromEntries: [LuaLibFeature.Iterator, LuaLibFeature.Symbol], diff --git a/src/lualib/Generator.ts b/src/lualib/Generator.ts new file mode 100644 index 000000000..37b639776 --- /dev/null +++ b/src/lualib/Generator.ts @@ -0,0 +1,31 @@ +interface GeneratorIterator { + ____coroutine: LuaThread; + [Symbol.iterator](): GeneratorIterator; + next: typeof __TS__GeneratorNext; +} + +function __TS__GeneratorIterator(this: GeneratorIterator) { + return this; +} + +function __TS__GeneratorNext(this: GeneratorIterator, ...args: Vararg) { + const co = this.____coroutine; + if (coroutine.status(co) === "dead") return { done: true }; + + const [status, value] = coroutine.resume(co, ...args); + if (!status) throw value; + + return { value, done: coroutine.status(co) === "dead" }; +} + +function __TS__Generator(this: void, fn: (this: void, ...args: any[]) => any) { + return function(this: void, ...args: Vararg): GeneratorIterator { + const argsLength = select("#", ...args); + return { + // Using explicit this there, since we don't pass arguments after the first nil and context is likely to be nil + ____coroutine: coroutine.create(() => fn((unpack || table.unpack)(args, 1, argsLength))), + [Symbol.iterator]: __TS__GeneratorIterator, + next: __TS__GeneratorNext, + }; + }; +} diff --git a/src/lualib/Iterator.ts b/src/lualib/Iterator.ts index ca4618d9f..ac1effadb 100644 --- a/src/lualib/Iterator.ts +++ b/src/lualib/Iterator.ts @@ -1,19 +1,32 @@ -function __TS__Iterator(this: void, iterable: Iterable): (this: void) => T { - if (iterable[Symbol.iterator]) { +/** @tupleReturn */ +function __TS__IteratorGeneratorStep(this: GeneratorIterator): [true, any] | [] { + const co = this.____coroutine; + + const [status, value] = coroutine.resume(co); + if (!status) throw value; + + if (coroutine.status(co) === "dead") return []; + return [true, value]; +} + +/** @tupleReturn */ +function __TS__IteratorIteratorStep(this: Iterator): [true, T] | [] { + const result = this.next(); + if (result.done) return []; + return [true, result.value]; +} + +/** @tupleReturn */ +function __TS__Iterator( + this: void, + iterable: Iterable | GeneratorIterator | readonly T[] +): [(...args: any[]) => [any, T] | [], ...any[]] { + if ("____coroutine" in iterable) { + return [__TS__IteratorGeneratorStep, iterable]; + } else if (iterable[Symbol.iterator]) { const iterator = iterable[Symbol.iterator](); - return () => { - const result = iterator.next(); - if (!result.done) { - return result.value; - } else { - return undefined; - } - }; + return [__TS__IteratorIteratorStep, iterator]; } else { - let i = 0; - return () => { - i += 1; - return iterable[i]; - }; + return ipairs(iterable as readonly T[]); } } diff --git a/src/lualib/declarations/coroutine.d.ts b/src/lualib/declarations/coroutine.d.ts new file mode 100644 index 000000000..1e648d6c8 --- /dev/null +++ b/src/lualib/declarations/coroutine.d.ts @@ -0,0 +1,19 @@ +/** @noSelfInFile */ + +interface LuaThread { + readonly __internal__: unique symbol; +} + +declare namespace coroutine { + function create(f: (...args: any[]) => any): LuaThread; + + /** @tupleReturn */ + function resume(co: LuaThread, ...val: any[]): [true, ...any[]] | [false, string]; + + function status(co: LuaThread): "running" | "suspended" | "normal" | "dead"; + + function wrap(f: (...args: any[]) => any): /** @tupleReturn */ (...args: any[]) => any[]; + + /** @tupleReturn */ + function yield(...args: any[]): any[]; +} diff --git a/src/lualib/declarations/global.d.ts b/src/lualib/declarations/global.d.ts index 35f2b3b3a..2e13fbdb9 100644 --- a/src/lualib/declarations/global.d.ts +++ b/src/lualib/declarations/global.d.ts @@ -24,3 +24,9 @@ declare function unpack(list: T[], i?: number, j?: number): T[]; declare function select(index: number, ...args: T[]): T; declare function select(index: "#", ...args: T[]): number; + +/** + * @luaIterator + * @tupleReturn + */ +declare function ipairs(t: Record): LuaTupleIterable<[number, T]>; diff --git a/src/transformation/visitors/class/members/method.ts b/src/transformation/visitors/class/members/method.ts index 4d9610d9b..6f6df45b0 100644 --- a/src/transformation/visitors/class/members/method.ts +++ b/src/transformation/visitors/class/members/method.ts @@ -1,9 +1,7 @@ import * as ts from "typescript"; import * as lua from "../../../../LuaAST"; import { TransformationContext } from "../../../context"; -import { ContextType, getFunctionContextType } from "../../../utils/function-context"; -import { createSelfIdentifier } from "../../../utils/lua-ast"; -import { transformFunctionBody, transformParameters } from "../../function"; +import { transformFunctionToExpression } from "../../function"; import { transformPropertyName } from "../../literal"; import { isStaticNode } from "../utils"; @@ -18,29 +16,17 @@ export function transformMethodDeclaration( return undefined; } + const methodTable = + isStaticNode(node) || noPrototype + ? lua.cloneIdentifier(className) + : lua.createTableIndexExpression(lua.cloneIdentifier(className), lua.createStringLiteral("prototype")); + let methodName = transformPropertyName(context, node.name); if (lua.isStringLiteral(methodName) && methodName.value === "toString") { methodName = lua.createStringLiteral("__tostring", node.name); } - const type = context.checker.getTypeAtLocation(node); - const functionContext = - getFunctionContextType(context, type) !== ContextType.Void ? createSelfIdentifier() : undefined; - const [paramNames, dots, restParamName] = transformParameters(context, node.parameters, functionContext); - - const [body] = transformFunctionBody(context, node.parameters, node.body, restParamName); - const functionExpression = lua.createFunctionExpression( - lua.createBlock(body), - paramNames, - dots, - lua.FunctionExpressionFlags.Declaration, - node.body - ); - - const methodTable = - isStaticNode(node) || noPrototype - ? lua.cloneIdentifier(className) - : lua.createTableIndexExpression(lua.cloneIdentifier(className), lua.createStringLiteral("prototype")); + const [functionExpression] = transformFunctionToExpression(context, node); return lua.createAssignmentStatement( lua.createTableIndexExpression(methodTable, methodName), diff --git a/src/transformation/visitors/function.ts b/src/transformation/visitors/function.ts index 7c82a89be..cfbd8622d 100644 --- a/src/transformation/visitors/function.ts +++ b/src/transformation/visitors/function.ts @@ -1,5 +1,6 @@ import * as ts from "typescript"; import * as lua from "../../LuaAST"; +import { assert } from "../../utils"; import { FunctionVisitor, TransformationContext } from "../context"; import { isVarargType } from "../utils/annotations"; import { createDefaultExportStringLiteral, hasDefaultExportModifier } from "../utils/export"; @@ -11,8 +12,8 @@ import { createSelfIdentifier, wrapInTable, } from "../utils/lua-ast"; +import { LuaLibFeature, transformLuaLibFunction } from "../utils/lualib"; import { peekScope, performHoisting, popScope, pushScope, Scope, ScopeType } from "../utils/scope"; -import { transformGeneratorFunctionBody } from "./generator"; import { transformIdentifier } from "./identifier"; import { transformBindingPattern } from "./variable-declaration"; @@ -157,12 +158,13 @@ export function transformParameters( return [paramNames, dotsLiteral, restParamName]; } -export function transformFunctionLikeDeclaration( - node: ts.FunctionLikeDeclaration, - context: TransformationContext -): lua.Expression { - const type = context.checker.getTypeAtLocation(node); +export function transformFunctionToExpression( + context: TransformationContext, + node: ts.FunctionLikeDeclaration +): [lua.Expression, Scope] { + assert(node.body); + const type = context.checker.getTypeAtLocation(node); let functionContext: lua.Identifier | undefined; if (getFunctionContextType(context, type) !== ContextType.Void) { if (ts.isArrowFunction(node)) { @@ -176,14 +178,10 @@ export function transformFunctionLikeDeclaration( } } - // Build parameter string - const [paramNames, dotsLiteral, spreadIdentifier] = transformParameters(context, node.parameters, functionContext); - let flags = lua.FunctionExpressionFlags.None; - - if (node.body === undefined) { - // This code can be reached only from object methods, which is TypeScript error - return lua.createNilLiteral(); + if (!ts.isBlock(node.body)) flags |= lua.FunctionExpressionFlags.Inline; + if (ts.isFunctionDeclaration(node) || ts.isMethodDeclaration(node)) { + flags |= lua.FunctionExpressionFlags.Declaration; } let body: ts.Block; @@ -193,14 +191,11 @@ export function transformFunctionLikeDeclaration( const returnExpression = ts.createReturn(node.body); body = ts.createBlock([returnExpression]); returnExpression.parent = body; - if (node.body) { - body.parent = node.body.parent; - } - flags |= lua.FunctionExpressionFlags.Inline; + if (node.body) body.parent = node.body.parent; } - const [transformedBody, scope] = transformFunctionBody(context, node.parameters, body, spreadIdentifier); - + const [paramNames, dotsLiteral, spreadIdentifier] = transformParameters(context, node.parameters, functionContext); + const [transformedBody, functionScope] = transformFunctionBody(context, node.parameters, body, spreadIdentifier); const functionExpression = lua.createFunctionExpression( lua.createBlock(transformedBody), paramNames, @@ -209,12 +204,31 @@ export function transformFunctionLikeDeclaration( node ); + return [ + node.asteriskToken + ? transformLuaLibFunction(context, LuaLibFeature.Generator, undefined, functionExpression) + : functionExpression, + functionScope, + ]; +} + +export function transformFunctionLikeDeclaration( + node: ts.FunctionLikeDeclaration, + context: TransformationContext +): lua.Expression { + if (node.body === undefined) { + // This code can be reached only from object methods, which is TypeScript error + return lua.createNilLiteral(); + } + + const [functionExpression, functionScope] = transformFunctionToExpression(context, node); + // Handle named function expressions which reference themselves - if (ts.isFunctionExpression(node) && node.name && scope.referencedSymbols) { + if (ts.isFunctionExpression(node) && node.name && functionScope.referencedSymbols) { const symbol = context.checker.getSymbolAtLocation(node.name); if (symbol) { // TODO: Not using symbol ids because of https://github.com/microsoft/TypeScript/issues/37131 - const isReferenced = [...scope.referencedSymbols].some(([, nodes]) => + const isReferenced = [...functionScope.referencedSymbols].some(([, nodes]) => nodes.some(n => context.checker.getSymbolAtLocation(n)?.valueDeclaration === symbol.valueDeclaration) ); @@ -234,27 +248,10 @@ export function transformFunctionLikeDeclaration( export const transformFunctionDeclaration: FunctionVisitor = (node, context) => { // Don't transform functions without body (overload declarations) - if (!node.body) { + if (node.body === undefined) { return undefined; } - const type = context.checker.getTypeAtLocation(node); - const functionContext = - getFunctionContextType(context, type) !== ContextType.Void ? createSelfIdentifier() : undefined; - const [params, dotsLiteral, restParamName] = transformParameters(context, node.parameters, functionContext); - - const [body, functionScope] = node.asteriskToken - ? transformGeneratorFunctionBody(context, node.parameters, node.body, restParamName) - : transformFunctionBody(context, node.parameters, node.body, restParamName); - - const block = lua.createBlock(body); - const functionExpression = lua.createFunctionExpression( - block, - params, - dotsLiteral, - lua.FunctionExpressionFlags.Declaration - ); - if (hasDefaultExportModifier(node)) { return lua.createAssignmentStatement( lua.createTableIndexExpression(createExportsIdentifier(), createDefaultExportStringLiteral(node)), @@ -262,6 +259,8 @@ export const transformFunctionDeclaration: FunctionVisitor = (expression, context) => + lua.createCallExpression( + lua.createTableIndexExpression(lua.createIdentifier("coroutine"), lua.createStringLiteral("yield")), + expression.expression ? [context.transformExpression(expression.expression)] : [], + expression + ); diff --git a/src/transformation/visitors/generator.ts b/src/transformation/visitors/generator.ts deleted file mode 100644 index 60e48e0d9..000000000 --- a/src/transformation/visitors/generator.ts +++ /dev/null @@ -1,114 +0,0 @@ -import * as ts from "typescript"; -import * as lua from "../../LuaAST"; -import { FunctionVisitor, TransformationContext } from "../context"; -import { wrapInTable } from "../utils/lua-ast"; -import { importLuaLibFeature, LuaLibFeature } from "../utils/lualib"; -import { Scope } from "../utils/scope"; -import { transformFunctionBody } from "./function"; - -export function transformGeneratorFunctionBody( - context: TransformationContext, - parameters: ts.NodeArray, - body: ts.Block, - spreadIdentifier?: lua.Identifier -): [lua.Statement[], Scope] { - importLuaLibFeature(context, LuaLibFeature.Symbol); - const [functionBody, functionScope] = transformFunctionBody(context, parameters, body); - - const coroutineIdentifier = lua.createIdentifier("____co"); - const valueIdentifier = lua.createIdentifier("____value"); - const errIdentifier = lua.createIdentifier("____err"); - const itIdentifier = lua.createIdentifier("____it"); - - // local ____co = coroutine.create(originalFunction) - const coroutine = lua.createVariableDeclarationStatement( - coroutineIdentifier, - lua.createCallExpression( - lua.createTableIndexExpression(lua.createIdentifier("coroutine"), lua.createStringLiteral("create")), - [lua.createFunctionExpression(lua.createBlock(functionBody))] - ) - ); - - const nextBody = []; - // coroutine.resume(__co, ...) - const resumeCall = lua.createCallExpression( - lua.createTableIndexExpression(lua.createIdentifier("coroutine"), lua.createStringLiteral("resume")), - [coroutineIdentifier, lua.createDotsLiteral()] - ); - - // ____err, ____value = coroutine.resume(____co, ...) - nextBody.push(lua.createVariableDeclarationStatement([errIdentifier, valueIdentifier], resumeCall)); - - // if(not ____err){error(____value)} - const errorCheck = lua.createIfStatement( - lua.createUnaryExpression(errIdentifier, lua.SyntaxKind.NotOperator), - lua.createBlock([ - lua.createExpressionStatement(lua.createCallExpression(lua.createIdentifier("error"), [valueIdentifier])), - ]) - ); - nextBody.push(errorCheck); - - // coroutine.status(____co) == "dead"; - const coStatus = lua.createCallExpression( - lua.createTableIndexExpression(lua.createIdentifier("coroutine"), lua.createStringLiteral("status")), - [coroutineIdentifier] - ); - const status = lua.createBinaryExpression( - coStatus, - lua.createStringLiteral("dead"), - lua.SyntaxKind.EqualityOperator - ); - - // {done = coroutine.status(____co) == "dead"; value = ____value} - const iteratorResult = lua.createTableExpression([ - lua.createTableFieldExpression(status, lua.createStringLiteral("done")), - lua.createTableFieldExpression(valueIdentifier, lua.createStringLiteral("value")), - ]); - nextBody.push(lua.createReturnStatement([iteratorResult])); - - // function(____, ...) - const nextFunctionDeclaration = lua.createFunctionExpression( - lua.createBlock(nextBody), - [lua.createAnonymousIdentifier()], - lua.createDotsLiteral() - ); - - // ____it = {next = function(____, ...)} - const iterator = lua.createVariableDeclarationStatement( - itIdentifier, - lua.createTableExpression([ - lua.createTableFieldExpression(nextFunctionDeclaration, lua.createStringLiteral("next")), - ]) - ); - - const symbolIterator = lua.createTableIndexExpression( - lua.createIdentifier("Symbol"), - lua.createStringLiteral("iterator") - ); - - const block = [ - coroutine, - iterator, - // ____it[Symbol.iterator] = {return ____it} - lua.createAssignmentStatement( - lua.createTableIndexExpression(itIdentifier, symbolIterator), - lua.createFunctionExpression(lua.createBlock([lua.createReturnStatement([itIdentifier])])) - ), - // return ____it - lua.createReturnStatement([itIdentifier]), - ]; - - if (spreadIdentifier) { - const spreadTable = wrapInTable(lua.createDotsLiteral()); - block.unshift(lua.createVariableDeclarationStatement(spreadIdentifier, spreadTable)); - } - - return [block, functionScope]; -} - -export const transformYieldExpression: FunctionVisitor = (expression, context) => - lua.createCallExpression( - lua.createTableIndexExpression(lua.createIdentifier("coroutine"), lua.createStringLiteral("yield")), - expression.expression ? [context.transformExpression(expression.expression)] : [], - expression - ); diff --git a/src/transformation/visitors/index.ts b/src/transformation/visitors/index.ts index 17b25ffd7..48469a84b 100644 --- a/src/transformation/visitors/index.ts +++ b/src/transformation/visitors/index.ts @@ -17,8 +17,7 @@ import { transformDeleteExpression } from "./delete"; import { transformEnumDeclaration } from "./enum"; import { transformThrowStatement, transformTryStatement } from "./errors"; import { transformExpressionStatement } from "./expression-statement"; -import { transformFunctionDeclaration, transformFunctionLikeDeclaration } from "./function"; -import { transformYieldExpression } from "./generator"; +import { transformFunctionDeclaration, transformFunctionLikeDeclaration, transformYieldExpression } from "./function"; import { transformIdentifierExpression } from "./identifier"; import { literalVisitors } from "./literal"; import { transformDoStatement, transformWhileStatement } from "./loops/do-while"; diff --git a/src/transformation/visitors/loops/for-of.ts b/src/transformation/visitors/loops/for-of.ts index 44656088a..67adb2ca2 100644 --- a/src/transformation/visitors/loops/for-of.ts +++ b/src/transformation/visitors/loops/for-of.ts @@ -143,7 +143,7 @@ function transformForOfIteratorStatement( context.transformExpression(statement.expression) ); - return lua.createForInStatement(block, [valueVariable], [iterable], statement); + return lua.createForInStatement(block, [lua.createAnonymousIdentifier(), valueVariable], [iterable], statement); } export const transformForOfStatement: FunctionVisitor = (node, context) => { diff --git a/test/unit/functions/functions.spec.ts b/test/unit/functions/functions.spec.ts index 73549d918..7e4aa6a08 100644 --- a/test/unit/functions/functions.spec.ts +++ b/test/unit/functions/functions.spec.ts @@ -34,20 +34,11 @@ test.each(["b => a = b", "b => a += b", "b => a -= b", "b => a *= b", "b => a /= } ); -test.each([{ inp: [] }, { inp: [5] }, { inp: [1, 2] }])("Arrow Default Values (%p)", ({ inp }) => { - // Default value is 3 for v1 - const v1 = inp.length > 0 ? inp[0] : 3; - // Default value is 4 for v2 - const v2 = inp.length > 1 ? inp[1] : 4; - - const callArgs = inp.join(","); - - const result = util.transpileAndExecute( - `let add = (a: number = 3, b: number = 4) => a+b; - return add(${callArgs});` - ); - - expect(result).toBe(v1 + v2); +test.each([{ args: [] }, { args: [1] }, { args: [1, 2] }])("Arrow default values (%p)", ({ args }) => { + util.testFunction` + const add = (a = 3, b = 4) => a + b; + return add(${util.formatCode(...args)}); + `.expectToMatchJsResult(); }); test("Function Expression", () => { @@ -351,52 +342,6 @@ test("Complex element access call statement", () => { `.expectToMatchJsResult(); }); -test.each([1, 2])("Generator functions value (%p)", iterations => { - util.testFunction` - function* seq(value: number) { - let a = yield value + 1; - return 42; - } - const gen = seq(0); - let ret: number; - for(let i = 0; i < ${iterations}; ++i) { - ret = gen.next(i).value; - } - return ret; - `.expectToMatchJsResult(); -}); - -test.each([1, 2])("Generator functions done (%p)", iterations => { - util.testFunction` - function* seq(value: number) { - let a = yield value + 1; - return 42; - } - const gen = seq(0); - let ret: boolean; - for(let i = 0; i < ${iterations}; ++i) { - ret = gen.next(i).done; - } - return ret; - `.expectToMatchJsResult(); -}); - -test("Generator for..of", () => { - util.testFunction` - function* seq() { - yield(1); - yield(2); - yield(3); - return 4; - } - let result = 0; - for(let i of seq()) { - result = result * 10 + i; - } - return result - `.expectToMatchJsResult(); -}); - test("Function local overriding export", () => { util.testModule` export const foo = 5; diff --git a/test/unit/functions/generators.spec.ts b/test/unit/functions/generators.spec.ts new file mode 100644 index 000000000..90fb67aa4 --- /dev/null +++ b/test/unit/functions/generators.spec.ts @@ -0,0 +1,96 @@ +import * as util from "../../util"; + +test("generator parameters", () => { + util.testFunction` + function* generator(value: number) { + yield value; + } + + return generator(5).next(); + `.expectToMatchJsResult(); +}); + +test(".next()", () => { + util.testFunction` + function* generator() { + yield 1; + yield 2; + return 3; + } + + const it = generator(); + return [it.next(), it.next(), it.next(), it.next()]; + `.expectToMatchJsResult(); +}); + +test(".next() with parameters", () => { + util.testFunction` + function* generator() { + return yield 0; + } + + const it = generator(); + return [it.next(1), it.next(2), it.next(3)]; + `.expectToMatchJsResult(); +}); + +test("for..of", () => { + util.testFunction` + function* generator() { + yield 1; + yield 2; + yield undefined; + return 3; + } + + const results = []; + for (const value of generator()) { + results.push({ value }); + } + return results; + `.expectToMatchJsResult(); +}); + +test("function expression", () => { + util.testFunction` + const generator = function*() { + return true; + } + + return generator().next(); + `.expectToMatchJsResult(); +}); + +test("class method", () => { + util.testFunction` + class A { + *generator() { + return true; + } + } + + return new A().generator().next(); + `.expectToMatchJsResult(); +}); + +test("object member", () => { + util.testFunction` + const a = { + *generator() { + return true; + } + } + + return a.generator().next(); + `.expectToMatchJsResult(); +}); + +test("hoisting", () => { + util.testFunction` + return generator().next(); + + function* generator() { + return true; + } + `.expectToMatchJsResult(); +});