Skip to content

Commit 58ae087

Browse files
vbilopavvbilopav
committed
feat(test-runner): script includes, named steps, per-file setup/teardown/connection
Script reuse — \i / \ir includes (psql syntax, paste semantics): - `\i path` (cwd-relative) / `\ir path` (relative to the including file) on its own line splices the included file's content in place, as if pasted: SQL statements, assertions (attributed to the included file), HTTP blocks (participate in _response_{n} numbering), and header annotations (a shared annotation "profile" attaches with one include line). - Recognized only between statements (never inside strings/dollar-quotes; no statement fragments — use functions/views). Nested, cycle-safe, depth-capped. Works in Setup/Teardown SqlFile steps. Errors report the included file's name and line. Quoted paths and a trailing ';' accepted; other backslash lines pass through to PostgreSQL untouched. Named steps (TestRunner.Steps): - name -> step dictionary (same idiom as ConnectionStrings/Profiles); Setup/Teardown arrays mix name references and inline step objects; unknown name is a configuration error (exit 3). Step logs show phase + name + resolved code: `setup step CreateTemplate: create database app_template_abc`. Per-file header annotations (leading -- comments, before the first statement): - `-- @setup Name [Name ...]` runs named steps before the file - `-- @teardown Name [Name ...]` runs after the file — always, best-effort, after the file's connection closes (so `drop database ... with (force)` works), on the run token (survives per-test timeout) - `-- @connection Name` runs the file, including its in-process endpoint calls, on a named ConnectionStrings entry - Repeatable; names whitespace- or comma-separated; accumulate and execute in written order (teardown not reversed); setup fail-fast, teardown best-effort. Together these enable per-test database isolation: clone a template, run in the clone, drop it (deterministic sequence ids). {rnd} indexed instances: - {rndN_1}..{rndN_9} are independent random tokens per length (all stable per run) for when several distinct same-length names are needed. Changelog updated (paste-model includes, named steps, per-file annotations, indexed tokens, example 21). Config synced across appsettings.json, ConfigTemplate.cs, ConfigDefaults.cs, ConfigSchemaGenerator.cs. Full suite green (2351) incl. 50 parser tests; examples 19/20/21 verified end-to-end (21: two parallel per-test clones from one template, shared annotation profile via \ir, Command step with {rnd} substitution).
1 parent 0a20ec9 commit 58ae087

14 files changed

Lines changed: 764 additions & 58 deletions

File tree

‎NpgsqlRestClient/Builder.cs‎

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3094,6 +3094,18 @@ public TestRunnerOptions BuildTestRunnerOptions()
30943094
}
30953095
}
30963096

3097+
// Named step registry (name → step), referenced by name from Setup/Teardown and test file headers.
3098+
foreach (var child in section.GetSection("Steps").GetChildren())
3099+
{
3100+
if (string.IsNullOrWhiteSpace(child.Key)) continue;
3101+
var step = ReadStep(child);
3102+
if (step is not null)
3103+
{
3104+
step.Name = child.Key;
3105+
opt.Steps[child.Key] = step;
3106+
}
3107+
}
3108+
30973109
opt.Setup = ReadTestSteps(section.GetSection("Setup"));
30983110
opt.Teardown = ReadTestSteps(section.GetSection("Teardown"));
30993111
return opt;
@@ -3106,21 +3118,43 @@ public TestRunnerOptions BuildTestRunnerOptions()
31063118
return string.IsNullOrWhiteSpace(s.Value) ? null : s.Value;
31073119
}
31083120

3121+
TestSetupStep? ReadStep(IConfigurationSection child)
3122+
{
3123+
var step = new TestSetupStep
3124+
{
3125+
Sql = _config.GetConfigStr("Sql", child),
3126+
SqlFile = _config.GetConfigStr("SqlFile", child),
3127+
Command = _config.GetConfigStr("Command", child),
3128+
WorkingDirectory = _config.GetConfigStr("WorkingDirectory", child),
3129+
ConnectionName = _config.GetConfigStr("ConnectionName", child),
3130+
};
3131+
return step.Sql is not null || step.SqlFile is not null || step.Command is not null ? step : null;
3132+
}
3133+
31093134
List<TestSetupStep> ReadTestSteps(IConfigurationSection stepsSection)
31103135
{
31113136
var list = new List<TestSetupStep>();
31123137
if (stepsSection.Exists() is false) return list;
31133138
foreach (var child in stepsSection.GetChildren())
31143139
{
3115-
var step = new TestSetupStep
3140+
// A string element is a reference to a named step from the Steps registry; an object element
3141+
// is an inline step. Both can be mixed in the same array.
3142+
if (string.IsNullOrWhiteSpace(child.Value) is false)
31163143
{
3117-
Sql = _config.GetConfigStr("Sql", child),
3118-
SqlFile = _config.GetConfigStr("SqlFile", child),
3119-
Command = _config.GetConfigStr("Command", child),
3120-
WorkingDirectory = _config.GetConfigStr("WorkingDirectory", child),
3121-
ConnectionName = _config.GetConfigStr("ConnectionName", child),
3122-
};
3123-
if (step.Sql is not null || step.SqlFile is not null || step.Command is not null)
3144+
var name = child.Value.Trim();
3145+
if (opt.Steps.TryGetValue(name, out var named))
3146+
{
3147+
list.Add(named);
3148+
}
3149+
else
3150+
{
3151+
throw new InvalidOperationException(
3152+
$"TestRunner Setup/Teardown references unknown step '{name}' — define it under TestRunner:Steps.");
3153+
}
3154+
continue;
3155+
}
3156+
var step = ReadStep(child);
3157+
if (step is not null)
31243158
{
31253159
list.Add(step);
31263160
}

‎NpgsqlRestClient/Config.cs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,9 +109,16 @@ public void Build(string[] args, string[] skip)
109109
// process and stable across the whole config — usable wherever {ENV} placeholders are
110110
// (connection strings, TestRunner Setup/Teardown). E.g. a test DB named per run:
111111
// "ConnectionStrings:Test": "...;Database=app_test_{rnd6};..." + "create database app_test_{rnd6}".
112+
// When several DISTINCT tokens of the same length are needed, indexed instances {rndN_1}..{rndN_9}
113+
// are each independent (e.g. {rnd3}, {rnd3_1} and {rnd3_2} are three different 3-char tokens).
112114
for (int n = 1; n <= 10; n++)
113115
{
114-
EnvDict[string.Concat("rnd", n.ToString(System.Globalization.CultureInfo.InvariantCulture))] = RandomToken(n);
116+
var nStr = n.ToString(System.Globalization.CultureInfo.InvariantCulture);
117+
EnvDict[string.Concat("rnd", nStr)] = RandomToken(n);
118+
for (int m = 1; m <= 9; m++)
119+
{
120+
EnvDict[string.Concat("rnd", nStr, "_", m.ToString(System.Globalization.CultureInfo.InvariantCulture))] = RandomToken(n);
121+
}
115122
}
116123
}
117124

‎NpgsqlRestClient/ConfigDefaults.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1123,6 +1123,7 @@ private static JsonObject GetTestRunnerDefaults()
11231123
["IsSuccess"] = "is_success"
11241124
}
11251125
},
1126+
["Steps"] = new JsonObject(),
11261127
["Setup"] = new JsonArray(),
11271128
["Teardown"] = new JsonArray()
11281129
};

‎NpgsqlRestClient/ConfigSchemaGenerator.cs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,7 @@ public static partial class ConfigSchemaGenerator
248248
["Stats:ActivityPath"] = "Path for current database activity.\nReturns data from pg_stat_activity showing active sessions, queries, and wait events.\nSecurity Consideration: Shows currently running queries which may contain sensitive data.",
249249
["TestRunner"] = "SQL test runner. Invoked with the `--test` command-line flag (otherwise inert).\nDiscovers *.test.sql files, runs each in an isolated non-pooled connection, can invoke endpoints in-process from an embedded `/* GET /path */` block (response captured into a temp table), and asserts via boolean-returning SELECTs or `do $$ ... assert ... $$;` blocks.\nExit codes: 0 pass, 1 failures, 2 errors, 3 config/runner error, 4 no tests found.",
250250
["TestRunner:FilePattern"] = "Glob (same engine as SqlFileSource) selecting *.test.sql files. Empty disables discovery.\nCo-located convention: app.sql (endpoint) next to app.test.sql (test).",
251-
["TestRunner:ConnectionName"] = "Optional: a ConnectionStrings entry to run the tests against instead of the app's main connection.\nIn test mode it becomes the connection used for endpoint type-checking (Describe) and execution, so it can point at a dedicated test database that a Setup step creates first (it need not exist at startup). Empty = use the main connection.\nTip: {rnd1}..{rnd10} are random lowercase tokens (length = the digit), stable for the whole run, usable in any connection string or Setup/Teardown SQL (e.g. Database=app_test_{rnd6}).",
251+
["TestRunner:ConnectionName"] = "Optional: a ConnectionStrings entry to run the tests against instead of the app's main connection.\nIn test mode it becomes the connection used for endpoint type-checking (Describe) and execution, so it can point at a dedicated test database that a Setup step creates first (it need not exist at startup). Empty = use the main connection.\nTip: {rnd1}..{rnd10} are random lowercase tokens (length = the digit), stable for the whole run, usable in any connection string or Setup/Teardown SQL (e.g. Database=app_test_{rnd6}). Need several distinct tokens of the same length? Indexed instances {rndN_1}..{rndN_9} are each independent.",
252252
["TestRunner:MaxParallelism"] = "Max test files run concurrently. 0 => processor count. Each test uses its own non-pooled connection.",
253253
["TestRunner:FailFast"] = "Stop scheduling new tests after the first failure/error (in-flight tests still finish).",
254254
["TestRunner:PerTestTimeout"] = "Per-test timeout. Accepts \"30s\", \"5m\", \"1h\", a plain number of seconds, or \"hh:mm:ss\". 0 disables.",
@@ -260,8 +260,9 @@ public static partial class ConfigSchemaGenerator
260260
["TestRunner:ResponseTempTable:Name"] = "Temp table name used when a test file has exactly ONE HTTP block.\nEach HTTP block gets its own fresh temp table (created without IF NOT EXISTS, so a duplicate name fails the test); writes are pg_temp-qualified. Per-block override: `# @response <name>`.",
261261
["TestRunner:ResponseTempTable:MultiNamePattern"] = "Temp table name pattern used when a test file has 2+ HTTP blocks.\n{n} is the 1-based block ordinal, e.g. \"_response_{n}\" => _response_1, _response_2, …",
262262
["TestRunner:ResponseTempTable:Columns"] = "Maps each response component to a column name. A null or empty name omits that column.",
263-
["TestRunner:Setup"] = "Run-once setup, BEFORE endpoint discovery. Steps run in the EXACT order written.\nEach entry is one of: { \"Command\": \"...\", \"WorkingDirectory\": \"...\" } (OS shell); { \"Sql\": \"...\" } | { \"SqlFile\": \"...\" } which run on the test connection, or set \"ConnectionName\" to run on another ConnectionStrings entry (e.g. an admin connection that runs `create database`).",
264-
["TestRunner:Teardown"] = "Run-once teardown, ALWAYS (best-effort), in the EXACT order written. Same step shapes as Setup (e.g. a `drop database` on an admin connection). The `Keep` option skips it.",
263+
["TestRunner:Steps"] = "Named, reusable steps (name → step; same shape as Setup/Teardown entries). Reference them by name in Setup/Teardown, or from an individual test file's leading header comments:\n-- @setup StepName [StepName ...] (runs before that file); -- @teardown StepName [StepName ...] (runs after that file, always, best-effort); -- @connection Name (runs that file on a named ConnectionStrings entry).\nAnnotations are repeatable; names may be whitespace- or comma-separated and run in the order written.\nTest files can also reuse SQL scripts in place with psql-style includes: `\\i file` (cwd-relative) or `\\ir file` (relative to the including file) — executed on the test's connection, inside its transaction.",
264+
["TestRunner:Setup"] = "Run-once setup, BEFORE endpoint discovery. Steps run in the EXACT order written.\nEach entry is a step NAME from \"Steps\", or an inline object: { \"Command\": \"...\", \"WorkingDirectory\": \"...\" } (OS shell); { \"Sql\": \"...\" } | { \"SqlFile\": \"...\" } which run on the test connection, or set \"ConnectionName\" to run on another ConnectionStrings entry (e.g. an admin connection that runs `create database`).",
265+
["TestRunner:Teardown"] = "Run-once teardown, ALWAYS (best-effort), in the EXACT order written. Same entries as Setup (step names or inline objects, e.g. a `drop database` on an admin connection). The `Keep` option skips it.",
265266
["CommandRetryOptions"] = "Command retry strategies and options for client and middleware commands.",
266267
["CommandRetryOptions:Strategies:default:RetrySequenceSeconds"] = "Retry sequence in seconds. Accepts decimal numbers (0.25 is quarter of a second). The length of the array determines the maximum number of retries.",
267268
["CommandRetryOptions:Strategies:default:ErrorCodes"] = "Error codes that will trigger a retry when executing a command. See https://www.postgresql.org/docs/current/errcodes-appendix.html",

‎NpgsqlRestClient/ConfigTemplate.cs‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1453,6 +1453,7 @@ public static partial class ConfigSchemaGenerator
14531453
// point at a dedicated test database that a Setup step creates first (it need not exist at startup).
14541454
// Empty = use the main connection. Tip: {rnd1}..{rnd10} are random lowercase tokens (length = the digit),
14551455
// stable for the whole run, usable in any connection string or Setup/Teardown SQL (e.g. Database=app_test_{rnd6}).
1456+
// Need several distinct tokens of the same length? Indexed instances {rndN_1}..{rndN_9} are each independent.
14561457
//
14571458
"ConnectionName": "",
14581459
//
@@ -1509,15 +1510,27 @@ public static partial class ConfigSchemaGenerator
15091510
}
15101511
},
15111512
//
1512-
// Run-once setup, BEFORE endpoint discovery. Steps run in the EXACT order written. Each entry is one of:
1513+
// Named, reusable steps (name → step; same shape as Setup/Teardown entries). Reference them by name in
1514+
// Setup/Teardown below, or from an individual test file's leading header comments:
1515+
// -- @setup StepName [StepName ...] runs before that file
1516+
// -- @teardown StepName [StepName ...] runs after that file (always, best-effort)
1517+
// -- @connection Name runs that file on a named ConnectionStrings entry
1518+
// Annotations are repeatable; names may be whitespace- or comma-separated and run in the order written.
1519+
// Test files can also reuse SQL scripts in place with psql-style includes: `\i file` (cwd-relative) or
1520+
// `\ir file` (relative to the including file) — executed on the test's connection, inside its transaction.
1521+
//
1522+
"Steps": {},
1523+
//
1524+
// Run-once setup, BEFORE endpoint discovery. Steps run in the EXACT order written. Each entry is a step
1525+
// NAME from "Steps" above, or an inline step object:
15131526
// { "Command": "...", "WorkingDirectory": "..." } — runs via the OS shell
15141527
// { "Sql": "..." } | { "SqlFile": "..." } — runs on the test connection; set "ConnectionName"
15151528
// to run on another ConnectionStrings entry (e.g.
15161529
// an admin connection that runs `create database`).
15171530
//
15181531
"Setup": [],
15191532
//
1520-
// Run-once teardown, ALWAYS (best-effort), in the EXACT order written. `Keep` skips it. Same step shapes as Setup.
1533+
// Run-once teardown, ALWAYS (best-effort), in the EXACT order written. `Keep` skips it. Same entries as Setup.
15211534
//
15221535
"Teardown": []
15231536
},

‎NpgsqlRestClient/Program.cs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -500,7 +500,18 @@
500500
var logTestNotices = options.LogConnectionNoticeEvents;
501501
options.LogConnectionNoticeEvents = false;
502502

503-
testRunner = new NpgsqlRestClient.Testing.TestRunner(options, builder.BuildTestRunnerOptions(), connectionString, builder.TestLogger, logTestNotices, builder.BuildTestRunnerConnectionStrings());
503+
try
504+
{
505+
testRunner = new NpgsqlRestClient.Testing.TestRunner(options, builder.BuildTestRunnerOptions(), connectionString, builder.TestLogger, logTestNotices, builder.BuildTestRunnerConnectionStrings());
506+
}
507+
catch (Exception ex)
508+
{
509+
// Configuration error (e.g. a Setup/Teardown entry referencing an unknown named step).
510+
new Out().Line($"Test runner configuration error: {ex.Message}", ConsoleColor.Red);
511+
builder.TestLogger?.LogError(ex, "Test runner configuration error");
512+
Environment.ExitCode = NpgsqlRestClient.Testing.TestRunner.ExitConfig;
513+
return;
514+
}
504515
if (await testRunner.SetupAsync() is false)
505516
{
506517
Environment.ExitCode = NpgsqlRestClient.Testing.TestRunner.ExitConfig;

‎NpgsqlRestClient/Testing/SqlTestFileParser.cs‎

Lines changed: 58 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,46 @@
33
namespace NpgsqlRestClient.Testing;
44

55
/// <summary>
6-
/// Splits a .test.sql file into an ordered list of steps (SQL statements and embedded HTTP requests),
7-
/// preserving interleaving. A char-by-char state machine handles ';' statement splitting, line/block
8-
/// comments, single quotes (with '' escapes), and dollar-quoted strings / DO blocks (so semicolons inside
9-
/// them don't split). A block comment whose first line is a request line becomes an <see cref="HttpStep"/>;
10-
/// any other comment is ignored.
6+
/// Splits a .test.sql file into an ordered list of steps (SQL statements, embedded HTTP requests, and
7+
/// include lines), preserving interleaving. A char-by-char state machine handles ';' statement splitting,
8+
/// line/block comments, single quotes (with '' escapes), and dollar-quoted strings / DO blocks (so
9+
/// semicolons inside them don't split). A block comment whose first line is a request line becomes an
10+
/// <see cref="HttpStep"/>; any other comment is ignored. A line starting with <c>\i</c>/<c>\ir</c> between
11+
/// statements becomes an <see cref="IncludeStep"/> (expanded by <see cref="TestFileLoader"/>).
1112
/// </summary>
1213
public static class SqlTestFileParser
1314
{
1415
private enum State { Normal, BlockComment, SingleQuote, DollarQuote }
1516

17+
/// <summary>
18+
/// Parses one line as an include: <c>\i path</c> (cwd-relative) or <c>\ir path</c> (relative to the
19+
/// including file). The path may be single-quoted; a trailing ';' is forgiven. Returns false for any
20+
/// other line (including other backslash commands).
21+
/// </summary>
22+
internal static bool TryParseIncludeLine(string line, out string path, out bool relativeToFile)
23+
{
24+
path = "";
25+
relativeToFile = false;
26+
var t = line.Trim();
27+
bool relative = t.StartsWith("\\ir", StringComparison.OrdinalIgnoreCase)
28+
&& (t.Length == 3 || char.IsWhiteSpace(t[3]));
29+
bool plain = !relative && t.StartsWith("\\i", StringComparison.OrdinalIgnoreCase)
30+
&& (t.Length == 2 || char.IsWhiteSpace(t[2]));
31+
if (!relative && !plain) return false;
32+
33+
var p = t[(relative ? 3 : 2)..].Trim();
34+
if (p.EndsWith(';')) p = p[..^1].TrimEnd(); // forgive a trailing ;
35+
if (p.Length > 1 && p[0] == '\'' && p[^1] == '\'')
36+
{
37+
p = p[1..^1]; // psql-style quoted path
38+
}
39+
if (p.Length == 0) return false;
40+
41+
path = p;
42+
relativeToFile = relative;
43+
return true;
44+
}
45+
1646
public static List<TestStep> Parse(string content)
1747
{
1848
var steps = new List<TestStep>();
@@ -63,6 +93,29 @@ void FlushSql()
6393
switch (state)
6494
{
6595
case State.Normal:
96+
// include: \i <path> (cwd-relative) or \ir <path> (relative to the including file) —
97+
// psql semantics. Recognized only between statements (pending statement is blank), so a
98+
// backslash inside SQL text/strings/dollar-quotes is never misread. Consumes the line.
99+
if (c == '\\' && !stmtHasContent)
100+
{
101+
int lineEnd = i;
102+
while (lineEnd < len && content[lineEnd] != '\n') lineEnd++;
103+
var includeLine = content[i..lineEnd].TrimEnd('\r').TrimEnd();
104+
if (TryParseIncludeLine(includeLine, out var path, out var relative))
105+
{
106+
steps.Add(new IncludeStep(path, relative, currentLine));
107+
i = lineEnd; // consume the include line; the newline is handled by the main loop
108+
continue;
109+
}
110+
// Not \i/\ir: keep just the backslash as SQL text and let normal parsing continue
111+
// (';' still splits; PostgreSQL will report the stray backslash clearly).
112+
stmt.Append(c);
113+
MarkContent();
114+
lastToken = "";
115+
lastTokenIsWord = false;
116+
i++;
117+
continue;
118+
}
66119
// line comment: -- (skip to end of line, never an HTTP step)
67120
if (c == '-' && i + 1 < len && content[i + 1] == '-')
68121
{

0 commit comments

Comments
 (0)