diff --git a/build/ci/templates/test_phases.yml b/build/ci/templates/test_phases.yml
index 026ed150740d..2e4cf0bf2a92 100644
--- a/build/ci/templates/test_phases.yml
+++ b/build/ci/templates/test_phases.yml
@@ -413,9 +413,10 @@ steps:
   # > npm run package
   # > npx gulp clean:cleanExceptTests
   # > npm run testSmoke
-  - script: |
+  - bash: |
+      npm install -g vsce
       npm run clean
-      npm run updateBuildNumber -- --buildNumber $TRAVIS_BUILD_NUMBER
+      npm run updateBuildNumber -- --buildNumber $BUILD_BUILDID
       npm run package
       npx gulp clean:cleanExceptTests
       npm run testSmoke
diff --git a/build/ci/vscode-python-pr-validation.yaml b/build/ci/vscode-python-pr-validation.yaml
index 052af7a6bda5..70cb8e2fab40 100644
--- a/build/ci/vscode-python-pr-validation.yaml
+++ b/build/ci/vscode-python-pr-validation.yaml
@@ -50,7 +50,7 @@ jobs:
       'Mac-Py3.7 Unit+Single':
         PythonVersion: '3.7'
         VMImageName: 'macos-10.13'
-        TestsToRun: 'testUnitTests, pythonUnitTests, testSingleWorkspace'
+        TestsToRun: 'testUnitTests, pythonUnitTests, testSingleWorkspace, testSmoke'
         NeedsPythonTestReqs: true
       'Mac-Py2.7 Unit+Single':
         PythonVersion: '2.7'
@@ -60,7 +60,7 @@ jobs:
       'Linux-Py3.7 Unit+Single':
         PythonVersion: '3.7'
         VMImageName: 'ubuntu-16.04'
-        TestsToRun: 'testUnitTests, pythonUnitTests, testSingleWorkspace'
+        TestsToRun: 'testUnitTests, pythonUnitTests, testSingleWorkspace, testSmoke'
         NeedsPythonTestReqs: true
       'Linux-Py2.7 Unit+Single':
         PythonVersion: '2.7'
diff --git a/news/3 Code Health/4515.md b/news/3 Code Health/4515.md
new file mode 100644
index 000000000000..96a01b5bc58a
--- /dev/null
+++ b/news/3 Code Health/4515.md	
@@ -0,0 +1 @@
+Corrected Smoke test failure for 'Run Python File In Terminal'
diff --git a/src/test/smoke/runInTerminal.smoke.test.ts b/src/test/smoke/runInTerminal.smoke.test.ts
index ba2efae820b1..013e6bff396d 100644
--- a/src/test/smoke/runInTerminal.smoke.test.ts
+++ b/src/test/smoke/runInTerminal.smoke.test.ts
@@ -43,7 +43,7 @@ suite('Smoke Test: Run Python File In Terminal', () => {
         }
         const textDocument = await openFile(file);
 
-        await vscode.commands.executeCommand<void>('python.execInTerminal', textDocument.uri, outputFile);
+        await vscode.commands.executeCommand<void>('python.execInTerminal', textDocument.uri);
         const checkIfFileHasBeenCreated = () => fs.pathExists(outputFile);
         await waitForCondition(checkIfFileHasBeenCreated, 30_000, `"${outputFile}" file not created`);
     });
diff --git a/src/testMultiRootWkspc/smokeTests/testExecInTerminal.py b/src/testMultiRootWkspc/smokeTests/testExecInTerminal.py
index 022a160544db..d83d46a740d9 100644
--- a/src/testMultiRootWkspc/smokeTests/testExecInTerminal.py
+++ b/src/testMultiRootWkspc/smokeTests/testExecInTerminal.py
@@ -3,11 +3,14 @@
 import os
 
 optlist, args = getopt.getopt(sys.argv, '')
-if len(args) < 2:
-    help_msg = '{} requires 1 parameter - the full path specification of the logfile to write.'.format(args[0])
-    raise RuntimeError(help_msg)
-    
-log_file = args[1]
+
+# If the caller has not specified the output file, create one for them with
+# the same name as the caller script, but with a .log extension.
+log_file = os.path.splitext(sys.argv[0])[0] + '.log'
+
+# If the output file is given, use that instead.
+if len(args) == 2:
+    log_file = args[1]
 
 with open(log_file, "a") as f:
     f.write(sys.executable)