Skip to content

Commit f416f47

Browse files
baldurkbaldurk
committed
Implement UI side of detecting vulkan layer problems and fixing as admin
* Stub functions for the actual detection/fixing part.
1 parent 206d9e0 commit f416f47

8 files changed

Lines changed: 316 additions & 43 deletions

File tree

‎qrenderdoc/Code/QRDUtils.cpp‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
#include <QMetaMethod>
3535
#include <QProcess>
3636
#include <QProgressDialog>
37+
#include <QStandardPaths>
3738
#include <QTreeWidget>
3839
#include <QtMath>
3940

@@ -673,6 +674,138 @@ class RDProgressDialog : public QProgressDialog
673674
static const int maxProgress = 1000;
674675
};
675676

677+
#if defined(Q_OS_WIN32)
678+
#include <shellapi.h>
679+
#include <windows.h>
680+
#endif
681+
682+
bool RunProcessAsAdmin(const QString &fullExecutablePath, const QStringList &params,
683+
std::function<void()> finishedCallback)
684+
{
685+
#if defined(Q_OS_WIN32)
686+
687+
std::wstring wideExe = fullExecutablePath.toStdWString();
688+
std::wstring wideParams = params.join(QChar(' ')).toStdWString();
689+
690+
SHELLEXECUTEINFOW info = {};
691+
info.cbSize = sizeof(info);
692+
info.fMask = SEE_MASK_NOCLOSEPROCESS;
693+
info.lpVerb = L"runas";
694+
info.lpFile = wideExe.c_str();
695+
info.lpParameters = wideParams.c_str();
696+
info.nShow = SW_SHOWNORMAL;
697+
698+
ShellExecuteExW(&info);
699+
700+
if((uintptr_t)info.hInstApp > 32 && info.hProcess != NULL)
701+
{
702+
if(finishedCallback)
703+
{
704+
HANDLE h = info.hProcess;
705+
706+
// do the wait on another thread
707+
LambdaThread *thread = new LambdaThread([h, finishedCallback]() {
708+
WaitForSingleObject(h, 30000);
709+
CloseHandle(h);
710+
GUIInvoke::call(finishedCallback);
711+
});
712+
thread->selfDelete(true);
713+
thread->start();
714+
}
715+
else
716+
{
717+
CloseHandle(info.hProcess);
718+
}
719+
720+
return true;
721+
}
722+
723+
return false;
724+
725+
#else
726+
// try to find a way to run the application elevated.
727+
const QString graphicalSudo[] = {
728+
"pkexec", "kdesudo", "gksudo", "beesu",
729+
};
730+
731+
// if none of the graphical options, then look for sudo and either
732+
const QString termEmulator[] = {
733+
"x-terminal-emulator", "gnome-terminal", "knosole", "xterm",
734+
};
735+
736+
for(const QString &sudo : graphicalSudo)
737+
{
738+
QString inPath = QStandardPaths::findExecutable(sudo);
739+
740+
// can't find in path
741+
if(inPath.isEmpty())
742+
continue;
743+
744+
QProcess *process = new QProcess;
745+
746+
QStringList sudoParams;
747+
sudoParams << fullExecutablePath;
748+
for(const QString &p : params)
749+
sudoParams << p;
750+
751+
qInfo() << "Running" << sudo << "with params" << sudoParams;
752+
753+
// run with sudo
754+
process->start(sudo, sudoParams);
755+
756+
// when the process exits, call the callback and delete
757+
QObject::connect(process, OverloadedSlot<int>::of(&QProcess::finished),
758+
[process, finishedCallback](int exitCode) {
759+
process->deleteLater();
760+
GUIInvoke::call(finishedCallback);
761+
});
762+
763+
return true;
764+
}
765+
766+
QString sudo = QStandardPaths::findExecutable("sudo");
767+
768+
if(sudo.isEmpty())
769+
{
770+
qCritical() << "Couldn't find graphical or terminal sudo program!\n"
771+
<< "Please run " << fullExecutablePath << "with args" << params << "manually.";
772+
return false;
773+
}
774+
775+
for(const QString &term : termEmulator)
776+
{
777+
QString inPath = QStandardPaths::findExecutable(term);
778+
779+
// can't find in path
780+
if(inPath.isEmpty())
781+
continue;
782+
783+
QProcess *process = new QProcess;
784+
785+
// run terminal sudo with emulator
786+
QStringList termParams;
787+
termParams << "-e"
788+
<< QString("bash -c 'sudo %1 %2'").arg(fullExecutablePath).arg(params.join(QChar(' ')));
789+
790+
process->start(term, termParams);
791+
792+
// when the process exits, call the callback and delete
793+
QObject::connect(process, OverloadedSlot<int>::of(&QProcess::finished),
794+
[process, finishedCallback](int exitCode) {
795+
process->deleteLater();
796+
GUIInvoke::call(finishedCallback);
797+
});
798+
799+
return true;
800+
}
801+
802+
qCritical() << "Couldn't find graphical or terminal emulator to launch sudo.\n"
803+
<< "Please run " << fullExecutablePath << "with args" << params << "manually.";
804+
805+
return false;
806+
#endif
807+
}
808+
676809
void ShowProgressDialog(QWidget *window, const QString &labelText, ProgressFinishedMethod finished,
677810
ProgressUpdateMethod update)
678811
{

‎qrenderdoc/Code/QRDUtils.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -687,6 +687,9 @@ class QProgressDialog;
687687
typedef std::function<float()> ProgressUpdateMethod;
688688
typedef std::function<bool()> ProgressFinishedMethod;
689689

690+
bool RunProcessAsAdmin(const QString &fullExecutablePath, const QStringList &params,
691+
std::function<void()> finishedCallback = std::function<void()>());
692+
690693
void ShowProgressDialog(QWidget *window, const QString &labelText, ProgressFinishedMethod finished,
691694
ProgressUpdateMethod update = ProgressUpdateMethod());
692695

‎qrenderdoc/Code/qrenderdoc.cpp‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,12 +64,24 @@ int main(int argc, char *argv[])
6464
temp = true;
6565
}
6666

67+
for(int i = 0; i < argc; i++)
68+
{
69+
if(!QString::compare(argv[i], "--install_vulkan_layer") && i + 1 < argc)
70+
{
71+
if(!QString::compare(argv[i + 1], "root"))
72+
RENDERDOC_UpdateVulkanLayerRegistration(true);
73+
else
74+
RENDERDOC_UpdateVulkanLayerRegistration(false);
75+
return 0;
76+
}
77+
}
78+
6779
QString remoteHost = "";
6880
uint remoteIdent = 0;
6981

7082
for(int i = 0; i + 1 < argc; i++)
7183
{
72-
if(!QString::compare(argv[i], "--REMOTEACCESS", Qt::CaseInsensitive))
84+
if(!QString::compare(argv[i], "--remoteaccess", Qt::CaseInsensitive))
7385
{
7486
QRegularExpression regexp("^([a-zA-Z0-9_-]+:)?([0-9]+)$");
7587

‎qrenderdoc/Windows/Dialogs/CaptureDialog.cpp‎

Lines changed: 128 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -157,8 +157,7 @@ CaptureDialog::CaptureDialog(CaptureContext &ctx, OnCaptureMethod captureCallbac
157157
// sort by PID by default
158158
ui->processList->sortByColumn(1, Qt::AscendingOrder);
159159

160-
// TODO Vulkan Layer
161-
ui->vulkanLayerWarn->setVisible(false);
160+
ui->vulkanLayerWarn->setVisible(RENDERDOC_NeedVulkanLayerRegistration(NULL, NULL, NULL));
162161

163162
m_CaptureCallback = captureCallback;
164163
m_InjectCallback = injectCallback;
@@ -280,7 +279,133 @@ void CaptureDialog::on_exePath_textChanged(const QString &text)
280279

281280
void CaptureDialog::on_vulkanLayerWarn_clicked()
282281
{
283-
// TODO Vulkan Layer
282+
QString caption = tr("Configure Vulkan layer settings in registry?");
283+
284+
uint32_t flags = 0;
285+
rdctype::array<rdctype::str> myJSONs;
286+
rdctype::array<rdctype::str> otherJSONs;
287+
288+
RENDERDOC_NeedVulkanLayerRegistration(&flags, &myJSONs, &otherJSONs);
289+
290+
const bool hasOtherJSON = (flags & eVulkan_OtherInstallsRegistered);
291+
const bool thisRegistered = (flags & eVulkan_ThisInstallRegistered);
292+
const bool needElevation = (flags & eVulkan_NeedElevation);
293+
const bool couldElevate = (flags & eVulkan_CouldElevate);
294+
const bool registerAll = (flags & eVulkan_RegisterAll);
295+
const bool updateAllowed = (flags & eVulkan_UpdateAllowed);
296+
297+
QString msg =
298+
tr("Vulkan capture happens through the API's layer mechanism. RenderDoc has detected that ");
299+
300+
if(hasOtherJSON)
301+
{
302+
if(otherJSONs.count > 1)
303+
msg +=
304+
tr("there are other RenderDoc builds registered already. They must be disabled so that "
305+
"capture can happen without nasty clashes.");
306+
else
307+
msg +=
308+
tr("there is another RenderDoc build registered already. It must be disabled so that "
309+
"capture can happen without nasty clashes.");
310+
311+
if(!thisRegistered)
312+
msg += tr(" Also ");
313+
}
314+
315+
if(!thisRegistered)
316+
{
317+
msg +=
318+
tr("the layer for this installation is not yet registered. This could be due to an "
319+
"upgrade from a version that didn't support Vulkan, or if this version is just a loose "
320+
"unzip/dev build.");
321+
}
322+
323+
msg += tr("\n\nWould you like to proceed with the following changes?\n\n");
324+
325+
if(hasOtherJSON)
326+
{
327+
for(const rdctype::str &j : otherJSONs)
328+
msg += (updateAllowed ? tr("Unregister/update: %1\n") : tr("Unregister: %1\n")).arg(ToQStr(j));
329+
330+
msg += "\n";
331+
}
332+
333+
if(!thisRegistered)
334+
{
335+
if(registerAll)
336+
{
337+
for(const rdctype::str &j : myJSONs)
338+
msg += (updateAllowed ? tr("Register/update: %1\n") : tr("Register: %1\n")).arg(ToQStr(j));
339+
}
340+
else
341+
{
342+
msg += updateAllowed ? tr("Register one of:\n") : tr("Register/update one of:\n");
343+
for(const rdctype::str &j : myJSONs)
344+
msg += tr(" -- %1\n").arg(ToQStr(j));
345+
}
346+
347+
msg += "\n";
348+
}
349+
350+
msg += tr("This is a one-off change, it won't be needed again unless the installation moves.");
351+
352+
QMessageBox::StandardButton install = RDDialog::question(this, caption, msg, RDDialog::YesNoCancel);
353+
354+
if(install == QMessageBox::Yes)
355+
{
356+
bool run = false;
357+
bool admin = false;
358+
359+
// if we need to elevate, just try it.
360+
if(needElevation)
361+
{
362+
admin = run = true;
363+
}
364+
// if we could elevate, ask the user
365+
else if(couldElevate)
366+
{
367+
QMessageBox::StandardButton elevate = RDDialog::question(
368+
this, tr("System layer install"),
369+
tr("Do you want to elevate permissions to install the layer at a system level?"),
370+
RDDialog::YesNoCancel);
371+
372+
if(elevate == QMessageBox::Yes)
373+
admin = true;
374+
else if(elevate == QMessageBox::No)
375+
admin = false;
376+
377+
run = (elevate != QMessageBox::Cancel);
378+
}
379+
// otherwise run non-elevated
380+
else
381+
{
382+
run = true;
383+
}
384+
385+
if(run)
386+
{
387+
if(admin)
388+
{
389+
RunProcessAsAdmin(qApp->applicationFilePath(), QStringList() << "--install_vulkan_layer"
390+
<< "root",
391+
[this]() {
392+
// ui->vulkanLayerWarn->setVisible(RENDERDOC_NeedVulkanLayerRegistration(NULL,
393+
// NULL, NULL));
394+
ui->vulkanLayerWarn->setVisible(false);
395+
});
396+
return;
397+
}
398+
else
399+
{
400+
QProcess process;
401+
process.start(qApp->applicationFilePath(), QStringList() << "--install_vulkan_layer"
402+
<< "user");
403+
process.waitForFinished(300);
404+
}
405+
}
406+
407+
ui->vulkanLayerWarn->setVisible(RENDERDOC_NeedVulkanLayerRegistration(NULL, NULL, NULL));
408+
}
284409
}
285410

286411
void CaptureDialog::on_processRefesh_clicked()

‎qrenderdoc/Windows/Dialogs/CaptureDialog.ui‎

Lines changed: 8 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -274,48 +274,17 @@
274274
<height>36</height>
275275
</size>
276276
</property>
277-
<property name="palette">
278-
<palette>
279-
<active>
280-
<colorrole role="Button">
281-
<brush brushstyle="SolidPattern">
282-
<color alpha="255">
283-
<red>255</red>
284-
<green>255</green>
285-
<blue>220</blue>
286-
</color>
287-
</brush>
288-
</colorrole>
289-
</active>
290-
<inactive>
291-
<colorrole role="Button">
292-
<brush brushstyle="SolidPattern">
293-
<color alpha="255">
294-
<red>255</red>
295-
<green>255</green>
296-
<blue>220</blue>
297-
</color>
298-
</brush>
299-
</colorrole>
300-
</inactive>
301-
<disabled>
302-
<colorrole role="Button">
303-
<brush brushstyle="SolidPattern">
304-
<color alpha="255">
305-
<red>255</red>
306-
<green>255</green>
307-
<blue>220</blue>
308-
</color>
309-
</brush>
310-
</colorrole>
311-
</disabled>
312-
</palette>
313-
</property>
314277
<property name="cursor">
315278
<cursorShape>PointingHandCursor</cursorShape>
316279
</property>
317-
<property name="autoFillBackground">
318-
<bool>true</bool>
280+
<property name="styleSheet">
281+
<string notr="true">QToolButton {
282+
background-color: #ffffdc;
283+
border: 1px solid black;
284+
}
285+
QToolButton:hover {
286+
background-color: #ddddbe;
287+
}</string>
319288
</property>
320289
<property name="text">
321290
<string>Warning: Vulkan capture is not configured.

‎renderdoc/api/replay/renderdoc_replay.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -642,6 +642,14 @@ RENDERDOC_ExecuteAndInject(const char *app, const char *workingDir, const char *
642642
extern "C" RENDERDOC_API uint32_t RENDERDOC_CC RENDERDOC_InjectIntoProcess(
643643
uint32_t pid, void *env, const char *logfile, const CaptureOptions *opts, bool32 waitForExit);
644644

645+
//////////////////////////////////////////////////////////////////////////
646+
// Vulkan layer handling
647+
//////////////////////////////////////////////////////////////////////////
648+
649+
extern "C" RENDERDOC_API bool RENDERDOC_CC RENDERDOC_NeedVulkanLayerRegistration(
650+
uint32_t *flags, rdctype::array<rdctype::str> *myJSONs, rdctype::array<rdctype::str> *otherJSONs);
651+
extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_UpdateVulkanLayerRegistration(bool elevate);
652+
645653
//////////////////////////////////////////////////////////////////////////
646654
// Miscellaneous!
647655
//////////////////////////////////////////////////////////////////////////

0 commit comments

Comments
 (0)