Skip to content
This repository was archived by the owner on Oct 23, 2023. It is now read-only.

Commit 37bfa17

Browse files
veluca93veluca93
committed
Implement execution groups on the backend and fix a use-after-free.
1 parent 4287294 commit 37bfa17

File tree

14 files changed

+531
-368
lines changed

14 files changed

+531
-368
lines changed

‎CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ include(GoogleTest)
4949
find_package(PythonInterp 3.5 REQUIRED)
5050

5151
# Enable colors in ninja
52-
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always -Wall -Wextra -Wno-unused-parameter")
52+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always -Wall -Wextra -Wno-unused-parameter -g")
5353
set(CMAKE_CXX_FLAGS_RELEASE "-O3")
5454

5555
# if hunter is disabled all these lines are noop

‎capnp/evaluation.capnp‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ struct Resources {
2626
stack @8 :UInt64; # 0 means unlimited
2727
}
2828

29-
struct Request {
29+
struct ProcessRequest {
3030
executable :union {
3131
system@0 :Text;
3232
localFile @1 :FileInfo;
@@ -37,12 +37,16 @@ struct Request {
3737
outputFiles @5 :List(Text); # Name of outputs
3838
limits @6 :Resources;
3939
extraTime @7 :Float32; # Time that should be added to the cpu time limit.
40-
exclusive @8 :Bool; # If set, no other execution should run at the same time.
40+
}
41+
42+
struct Request {
43+
processes @0 :List(ProcessRequest);
44+
exclusive @1 :Bool; # If set, no other execution should run at the same time.
4145

4246
# TODO: FIFOs
4347
}
4448

45-
struct Result {
49+
struct ProcessResult {
4650
status :union {
4751
success @0 :Void;
4852
signal @1 :UInt32;
@@ -60,6 +64,10 @@ struct Result {
6064
outputFiles @11 :List(FileInfo); # Name and hash of other outputs
6165
}
6266

67+
struct Result {
68+
processes @0 :List(ProcessResult);
69+
}
70+
6371
interface Evaluator extends(FileSender) {
6472
evaluate @0 (request :Request) -> (result :Result);
6573
id @1 () -> (text :Result);

‎capnp/server.capnp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
using import "file.capnp".FileSender;
44
using import "file.capnp".FileReceiver;
55
using import "sha256.capnp".SHA256;
6-
using import "evaluation.capnp".Result;
6+
using import "evaluation.capnp".ProcessResult;
77
using import "evaluation.capnp".Resources;
88
using import "evaluation.capnp".Evaluator;
99
using Cxx = import "/capnp/c++.capnp";
@@ -39,7 +39,7 @@ interface Execution {
3939

4040
# The following methods will only complete (i.e. return or call callbacks)
4141
# when the evaluation is complete.
42-
getResult @12 () -> (result :Result);
42+
getResult @12 () -> (result :ProcessResult);
4343
}
4444

4545
interface FrontendContext {

‎cpp/frontend/frontend.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ struct Resources {
2727
};
2828

2929
struct Result {
30-
capnproto::Result::Status::Which status;
30+
capnproto::ProcessResult::Status::Which status;
3131
uint32_t signal;
3232
uint32_t return_code;
3333
std::string error;

‎cpp/frontend/python/frontend.cpp‎

Lines changed: 28 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -42,17 +42,21 @@ PYBIND11_MODULE(task_maker_frontend, m) {
4242
.def_readwrite("memlock", &frontend::Resources::memlock)
4343
.def_readwrite("stack", &frontend::Resources::stack);
4444

45-
pybind11::enum_<capnproto::Result::Status::Which>(m, "ResultStatus")
46-
.value("SUCCESS", capnproto::Result::Status::Which::SUCCESS)
47-
.value("SIGNAL", capnproto::Result::Status::Which::SIGNAL)
48-
.value("RETURN_CODE", capnproto::Result::Status::Which::RETURN_CODE)
49-
.value("TIME_LIMIT", capnproto::Result::Status::Which::TIME_LIMIT)
50-
.value("WALL_LIMIT", capnproto::Result::Status::Which::WALL_LIMIT)
51-
.value("MEMORY_LIMIT", capnproto::Result::Status::Which::MEMORY_LIMIT)
52-
.value("MISSING_FILES", capnproto::Result::Status::Which::MISSING_FILES)
53-
.value("INTERNAL_ERROR", capnproto::Result::Status::Which::INTERNAL_ERROR)
45+
pybind11::enum_<capnproto::ProcessResult::Status::Which>(m, "ResultStatus")
46+
.value("SUCCESS", capnproto::ProcessResult::Status::Which::SUCCESS)
47+
.value("SIGNAL", capnproto::ProcessResult::Status::Which::SIGNAL)
48+
.value("RETURN_CODE",
49+
capnproto::ProcessResult::Status::Which::RETURN_CODE)
50+
.value("TIME_LIMIT", capnproto::ProcessResult::Status::Which::TIME_LIMIT)
51+
.value("WALL_LIMIT", capnproto::ProcessResult::Status::Which::WALL_LIMIT)
52+
.value("MEMORY_LIMIT",
53+
capnproto::ProcessResult::Status::Which::MEMORY_LIMIT)
54+
.value("MISSING_FILES",
55+
capnproto::ProcessResult::Status::Which::MISSING_FILES)
56+
.value("INTERNAL_ERROR",
57+
capnproto::ProcessResult::Status::Which::INTERNAL_ERROR)
5458
.value("MISSING_EXECUTABLE",
55-
capnproto::Result::Status::Which::MISSING_EXECUTABLE);
59+
capnproto::ProcessResult::Status::Which::MISSING_EXECUTABLE);
5660

5761
pybind11::class_<frontend::Result>(m, "Result")
5862
.def_readonly("status", &frontend::Result::status)
@@ -62,21 +66,27 @@ PYBIND11_MODULE(task_maker_frontend, m) {
6266
.def_readonly("resources", &frontend::Result::resources)
6367
.def("__repr__", [](const frontend::Result& res) {
6468
std::string message = "<Result ";
65-
if (res.status == capnproto::Result::Status::Which::SUCCESS)
69+
if (res.status == capnproto::ProcessResult::Status::Which::SUCCESS)
6670
message += "SUCCESS";
67-
else if (res.status == capnproto::Result::Status::Which::SIGNAL)
71+
else if (res.status == capnproto::ProcessResult::Status::Which::SIGNAL)
6872
message += "SIGNAL " + std::to_string(res.signal);
69-
else if (res.status == capnproto::Result::Status::Which::RETURN_CODE)
73+
else if (res.status ==
74+
capnproto::ProcessResult::Status::Which::RETURN_CODE)
7075
message += "RETURN_CODE " + std::to_string(res.return_code);
71-
else if (res.status == capnproto::Result::Status::Which::TIME_LIMIT)
76+
else if (res.status ==
77+
capnproto::ProcessResult::Status::Which::TIME_LIMIT)
7278
message += "TIME_LIMIT";
73-
else if (res.status == capnproto::Result::Status::Which::WALL_LIMIT)
79+
else if (res.status ==
80+
capnproto::ProcessResult::Status::Which::WALL_LIMIT)
7481
message += "WALL_LIMIT";
75-
else if (res.status == capnproto::Result::Status::Which::MEMORY_LIMIT)
82+
else if (res.status ==
83+
capnproto::ProcessResult::Status::Which::MEMORY_LIMIT)
7684
message += "MEMORY_LIMIT";
77-
else if (res.status == capnproto::Result::Status::Which::MISSING_FILES)
85+
else if (res.status ==
86+
capnproto::ProcessResult::Status::Which::MISSING_FILES)
7887
message += "MISSING_FILES";
79-
else if (res.status == capnproto::Result::Status::Which::INTERNAL_ERROR)
88+
else if (res.status ==
89+
capnproto::ProcessResult::Status::Which::INTERNAL_ERROR)
8090
message += "INTERNAL_ERROR";
8191
else
8292
message += "UNKNOWN";

‎cpp/sandbox/sandbox.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ struct ExecutionOptions {
3535
char executable[str_len] = {};
3636
bool prepare_executable = false;
3737
ExecutionOptions(std::string root_, std::string executable_) {
38+
memset(this, 0, sizeof(*this));
3839
stringcpy(root, root_.c_str());
3940
stringcpy(executable, executable_.c_str());
4041
strcpy(&args[0][0], executable);

‎cpp/server/cache.cpp‎

Lines changed: 115 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -16,119 +16,132 @@ inline void hash_combine(std::size_t& hash, const T& v) {
1616
hash ^= hasher(v) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
1717
}
1818

19-
uint64_t RequestHasher::operator()(capnproto::Request::Reader reader) const {
19+
uint64_t RequestHasher::operator()(capnproto::Request::Reader reader_) const {
2020
size_t hash = 0;
21-
hash_combine(hash, reader.getExecutable().which());
22-
switch (reader.getExecutable().which()) {
23-
case capnproto::Request::Executable::SYSTEM:
24-
hash_combine(hash, std::string(reader.getExecutable().getSystem()));
25-
break;
26-
case capnproto::Request::Executable::LOCAL_FILE:
27-
hash_combine(
28-
hash, std::string(reader.getExecutable().getLocalFile().getName()));
29-
hash_combine(
30-
hash, util::SHA256_t(reader.getExecutable().getLocalFile().getHash())
31-
.Hex());
32-
break;
33-
}
34-
for (std::string arg : reader.getArgs()) {
35-
hash_combine(hash, arg);
36-
}
37-
hash_combine(hash, util::SHA256_t(reader.getStdin()).Hex());
38-
for (auto in : reader.getInputFiles()) {
39-
hash_combine(hash, std::string(in.getName()));
40-
hash_combine(hash, util::SHA256_t(in.getHash()).Hex());
41-
hash_combine(hash, in.getExecutable());
42-
}
43-
for (std::string out : reader.getOutputFiles()) {
44-
hash_combine(hash, out);
45-
}
46-
hash_combine(hash, reader.getLimits().getCpuTime());
47-
hash_combine(hash, reader.getLimits().getWallTime());
48-
hash_combine(hash, reader.getLimits().getMemory());
49-
hash_combine(hash, reader.getLimits().getNproc());
50-
hash_combine(hash, reader.getLimits().getNofiles());
51-
hash_combine(hash, reader.getLimits().getFsize());
52-
hash_combine(hash, reader.getLimits().getMemlock());
53-
hash_combine(hash, reader.getLimits().getStack());
54-
hash_combine(hash, reader.getExclusive());
55-
hash_combine(hash, reader.getExtraTime());
21+
for (auto reader : reader_.getProcesses()) {
22+
hash_combine(hash, reader.getExecutable().which());
23+
switch (reader.getExecutable().which()) {
24+
case capnproto::ProcessRequest::Executable::SYSTEM:
25+
hash_combine(hash, std::string(reader.getExecutable().getSystem()));
26+
break;
27+
case capnproto::ProcessRequest::Executable::LOCAL_FILE:
28+
hash_combine(
29+
hash, std::string(reader.getExecutable().getLocalFile().getName()));
30+
hash_combine(hash, util::SHA256_t(
31+
reader.getExecutable().getLocalFile().getHash())
32+
.Hex());
33+
break;
34+
}
35+
for (std::string arg : reader.getArgs()) {
36+
hash_combine(hash, arg);
37+
}
38+
hash_combine(hash, util::SHA256_t(reader.getStdin()).Hex());
39+
for (auto in : reader.getInputFiles()) {
40+
hash_combine(hash, std::string(in.getName()));
41+
hash_combine(hash, util::SHA256_t(in.getHash()).Hex());
42+
hash_combine(hash, in.getExecutable());
43+
}
44+
for (std::string out : reader.getOutputFiles()) {
45+
hash_combine(hash, out);
46+
}
47+
hash_combine(hash, reader.getLimits().getCpuTime());
48+
hash_combine(hash, reader.getLimits().getWallTime());
49+
hash_combine(hash, reader.getLimits().getMemory());
50+
hash_combine(hash, reader.getLimits().getNproc());
51+
hash_combine(hash, reader.getLimits().getNofiles());
52+
hash_combine(hash, reader.getLimits().getFsize());
53+
hash_combine(hash, reader.getLimits().getMemlock());
54+
hash_combine(hash, reader.getLimits().getStack());
55+
hash_combine(hash, reader.getExtraTime());
56+
}
57+
hash_combine(hash, reader_.getExclusive());
5658
return hash;
5759
}
58-
bool RequestComparator::operator()(capnproto::Request::Reader a,
59-
capnproto::Request::Reader b) const {
60-
if (a.getExecutable().which() != b.getExecutable().which()) return false;
61-
switch (a.getExecutable().which()) {
62-
case capnproto::Request::Executable::SYSTEM:
63-
if (a.getExecutable().getSystem() != b.getExecutable().getSystem())
64-
return false;
65-
break;
66-
case capnproto::Request::Executable::LOCAL_FILE:
67-
if (a.getExecutable().getLocalFile().getName() !=
68-
b.getExecutable().getLocalFile().getName())
69-
return false;
70-
if (util::SHA256_t(a.getExecutable().getLocalFile().getHash()).Hex() !=
71-
util::SHA256_t(b.getExecutable().getLocalFile().getHash()).Hex())
72-
return false;
73-
break;
74-
}
75-
auto aargs = a.getArgs();
76-
auto bargs = b.getArgs();
77-
for (size_t i = 0; i < aargs.size(); i++) {
78-
if (aargs[i] != bargs[i]) return false;
79-
}
80-
if (util::SHA256_t(a.getStdin()).Hex() != util::SHA256_t(b.getStdin()).Hex())
81-
return false;
82-
std::vector<std::tuple<std::string, std::string, bool>> ainput;
83-
std::vector<std::tuple<std::string, std::string, bool>> binput;
84-
for (auto in : a.getInputFiles()) {
85-
ainput.emplace_back(in.getName(), util::SHA256_t(in.getHash()).Hex(),
86-
in.getExecutable());
87-
}
88-
for (auto in : b.getInputFiles()) {
89-
binput.emplace_back(in.getName(), util::SHA256_t(in.getHash()).Hex(),
90-
in.getExecutable());
91-
}
92-
std::sort(ainput.begin(), ainput.end());
93-
std::sort(binput.begin(), binput.end());
94-
if (ainput != binput) return false;
95-
std::vector<std::string> aoutput;
96-
std::vector<std::string> boutput;
97-
for (auto out : a.getOutputFiles()) {
98-
aoutput.emplace_back(out);
99-
}
100-
for (auto out : b.getOutputFiles()) {
101-
boutput.emplace_back(out);
60+
bool RequestComparator::operator()(capnproto::Request::Reader a_,
61+
capnproto::Request::Reader b_) const {
62+
if (a_.getProcesses().size() != b_.getProcesses().size()) return false;
63+
if (a_.getExclusive() != b_.getExclusive()) return false;
64+
for (size_t i = 0; i < a_.getProcesses().size(); i++) {
65+
auto a = a_.getProcesses()[i];
66+
auto b = b_.getProcesses()[i];
67+
if (a.getExecutable().which() != b.getExecutable().which()) return false;
68+
switch (a.getExecutable().which()) {
69+
case capnproto::ProcessRequest::Executable::SYSTEM:
70+
if (a.getExecutable().getSystem() != b.getExecutable().getSystem())
71+
return false;
72+
break;
73+
case capnproto::ProcessRequest::Executable::LOCAL_FILE:
74+
if (a.getExecutable().getLocalFile().getName() !=
75+
b.getExecutable().getLocalFile().getName())
76+
return false;
77+
if (util::SHA256_t(a.getExecutable().getLocalFile().getHash()).Hex() !=
78+
util::SHA256_t(b.getExecutable().getLocalFile().getHash()).Hex())
79+
return false;
80+
break;
81+
}
82+
auto aargs = a.getArgs();
83+
auto bargs = b.getArgs();
84+
for (size_t i = 0; i < aargs.size(); i++) {
85+
if (aargs[i] != bargs[i]) return false;
86+
}
87+
if (util::SHA256_t(a.getStdin()).Hex() !=
88+
util::SHA256_t(b.getStdin()).Hex())
89+
return false;
90+
std::vector<std::tuple<std::string, std::string, bool>> ainput;
91+
std::vector<std::tuple<std::string, std::string, bool>> binput;
92+
for (auto in : a.getInputFiles()) {
93+
ainput.emplace_back(in.getName(), util::SHA256_t(in.getHash()).Hex(),
94+
in.getExecutable());
95+
}
96+
for (auto in : b.getInputFiles()) {
97+
binput.emplace_back(in.getName(), util::SHA256_t(in.getHash()).Hex(),
98+
in.getExecutable());
99+
}
100+
std::sort(ainput.begin(), ainput.end());
101+
std::sort(binput.begin(), binput.end());
102+
if (ainput != binput) return false;
103+
std::vector<std::string> aoutput;
104+
std::vector<std::string> boutput;
105+
for (auto out : a.getOutputFiles()) {
106+
aoutput.emplace_back(out);
107+
}
108+
for (auto out : b.getOutputFiles()) {
109+
boutput.emplace_back(out);
110+
}
111+
std::sort(aoutput.begin(), aoutput.end());
112+
std::sort(boutput.begin(), boutput.end());
113+
if (aoutput != boutput) return false;
114+
if (a.getLimits().getCpuTime() != b.getLimits().getCpuTime()) return false;
115+
if (a.getLimits().getWallTime() != b.getLimits().getWallTime())
116+
return false;
117+
if (a.getLimits().getMemory() != b.getLimits().getMemory()) return false;
118+
if (a.getLimits().getNproc() != b.getLimits().getNproc()) return false;
119+
if (a.getLimits().getNofiles() != b.getLimits().getNofiles()) return false;
120+
if (a.getLimits().getFsize() != b.getLimits().getFsize()) return false;
121+
if (a.getLimits().getMemory() != b.getLimits().getMemory()) return false;
122+
if (a.getLimits().getStack() != b.getLimits().getStack()) return false;
123+
if (a.getExtraTime() != b.getExtraTime()) return false;
102124
}
103-
std::sort(aoutput.begin(), aoutput.end());
104-
std::sort(boutput.begin(), boutput.end());
105-
if (aoutput != boutput) return false;
106-
if (a.getLimits().getCpuTime() != b.getLimits().getCpuTime()) return false;
107-
if (a.getLimits().getWallTime() != b.getLimits().getWallTime()) return false;
108-
if (a.getLimits().getMemory() != b.getLimits().getMemory()) return false;
109-
if (a.getLimits().getNproc() != b.getLimits().getNproc()) return false;
110-
if (a.getLimits().getNofiles() != b.getLimits().getNofiles()) return false;
111-
if (a.getLimits().getFsize() != b.getLimits().getFsize()) return false;
112-
if (a.getLimits().getMemory() != b.getLimits().getMemory()) return false;
113-
if (a.getLimits().getStack() != b.getLimits().getStack()) return false;
114-
if (a.getExclusive() != b.getExclusive()) return false;
115-
if (a.getExtraTime() != b.getExtraTime()) return false;
116125
return true;
117126
}
118127

119-
std::vector<util::SHA256_t> Hashes(capnproto::Request::Reader req,
120-
capnproto::Result::Reader res) {
128+
std::vector<util::SHA256_t> Hashes(capnproto::Request::Reader req_,
129+
capnproto::Result::Reader res_) {
121130
std::vector<util::SHA256_t> ans;
122131
auto add = [&ans](util::SHA256_t hash) {
123132
if (!hash.isZero()) ans.push_back(hash);
124133
};
125-
add(req.getStdin());
126-
if (req.getExecutable().isLocalFile())
127-
add(req.getExecutable().getLocalFile().getHash());
128-
for (auto f : req.getInputFiles()) add(f.getHash());
129-
add(res.getStdout());
130-
add(res.getStderr());
131-
for (auto f : res.getOutputFiles()) add(f.getHash());
134+
for (auto req : req_.getProcesses()) {
135+
add(req.getStdin());
136+
if (req.getExecutable().isLocalFile())
137+
add(req.getExecutable().getLocalFile().getHash());
138+
for (auto f : req.getInputFiles()) add(f.getHash());
139+
}
140+
for (auto res : res_.getProcesses()) {
141+
add(res.getStdout());
142+
add(res.getStderr());
143+
for (auto f : res.getOutputFiles()) add(f.getHash());
144+
}
132145
return ans;
133146
};
134147

0 commit comments

Comments
 (0)