Skip to content
This repository was archived by the owner on Oct 23, 2023. It is now read-only.

Commit 54510cc

Browse files
edomora97edomora97
committed
Statements compilation
For now only the IOI-like tasks are supported, using the template from the Italian Olympiad in Informatics which is taken from cmsbooklet (https://github.com/algorithm-ninja/cmsbooklet)
1 parent b9a0a0d commit 54510cc

27 files changed

Lines changed: 1555 additions & 46 deletions

‎cpp/frontend/frontend.cpp‎

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -71,19 +71,19 @@ std::unique_ptr<Fifo> Fifo::New(kj::Promise<T>&& p) {
7171
namespace {
7272
class FileProvider : public capnproto::FileSender::Server {
7373
public:
74-
explicit FileProvider(
75-
std::unordered_map<util::SHA256_t, std::string, util::SHA256_t::Hasher>
76-
known_files)
74+
explicit FileProvider(std::unordered_map<util::SHA256_t, util::FileWrapper,
75+
util::SHA256_t::Hasher>
76+
known_files)
7777
: known_files_(std::move(known_files)) {}
7878

7979
kj::Promise<void> requestFile(RequestFileContext context) override {
80-
std::string path = known_files_.at(context.getParams().getHash());
81-
return util::File::HandleRequestFile(path,
80+
util::FileWrapper *file = &known_files_.at(context.getParams().getHash());
81+
return util::File::HandleRequestFile(file,
8282
context.getParams().getReceiver());
8383
}
8484

8585
private:
86-
std::unordered_map<util::SHA256_t, std::string, util::SHA256_t::Hasher>
86+
std::unordered_map<util::SHA256_t, util::FileWrapper, util::SHA256_t::Hasher>
8787
known_files_;
8888
};
8989

@@ -159,7 +159,24 @@ File* Frontend::provideFile(const std::string& path,
159159
bool is_executable) {
160160
auto req = frontend_context_.provideFileRequest();
161161
util::SHA256_t hash = util::File::Hash(path);
162-
known_files_.emplace(hash, path);
162+
known_files_.emplace(hash, util::FileWrapper::FromPath(path));
163+
hash.ToCapnp(req.initHash());
164+
req.setDescription(description);
165+
req.setIsExecutable(is_executable);
166+
files_.push_back(File::New(req.send(), this, is_executable));
167+
return files_.back().get();
168+
}
169+
170+
File *Frontend::provideFileContent(const std::string &content,
171+
const std::string &description,
172+
bool is_executable) {
173+
auto req = frontend_context_.provideFileRequest();
174+
util::SHA256 hasher;
175+
// NOLINTNEXTLINE
176+
hasher.update(reinterpret_cast<const unsigned char *>(&content[0]),
177+
content.size());
178+
util::SHA256_t hash = hasher.finalize();
179+
known_files_.emplace(hash, util::FileWrapper::FromContent(content));
163180
hash.ToCapnp(req.initHash());
164181
req.setDescription(description);
165182
req.setIsExecutable(is_executable);
@@ -188,7 +205,7 @@ ExecutionGroup* Frontend::addExecutionGroup(const std::string& description) {
188205
void Frontend::evaluate() {
189206
finish_builder_.AddPromise(std::move(builder_).Finalize().then([this]() {
190207
auto req = frontend_context_.startEvaluationRequest();
191-
req.setSender(kj::heap<FileProvider>(known_files_));
208+
req.setSender(kj::heap<FileProvider>(std::move(known_files_)));
192209
return req.send().ignoreResult();
193210
}),
194211
"Evaluate");

‎cpp/frontend/frontend.hpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,11 @@ class Frontend {
200200
File* provideFile(const std::string& path, const std::string& description,
201201
bool is_executable);
202202

203+
// Defines a file that is provided by the frontend, loading it from its
204+
// content.
205+
File *provideFileContent(const std::string &content,
206+
const std::string &description, bool is_executable);
207+
203208
// Creates a new execution with the given description.
204209
Execution* addExecution(const std::string& description);
205210

@@ -217,7 +222,7 @@ class Frontend {
217222
private:
218223
capnp::EzRpcClient client_;
219224
capnproto::FrontendContext::Client frontend_context_;
220-
std::unordered_map<util::SHA256_t, std::string, util::SHA256_t::Hasher>
225+
std::unordered_map<util::SHA256_t, util::FileWrapper, util::SHA256_t::Hasher>
221226
known_files_;
222227
util::UnionPromiseBuilder builder_;
223228
util::UnionPromiseBuilder finish_builder_;

‎cpp/frontend/python/frontend.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
#pragma GCC diagnostic push
22
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
33
#include <pybind11/functional.h>
4-
#include <pybind11/pybind11.h>
54
#include <pybind11/stl.h>
65
#pragma GCC diagnostic pop
76

@@ -199,6 +198,9 @@ PYBIND11_MODULE(task_maker_frontend, m) {
199198
.def("provideFile", &frontend::Frontend::provideFile,
200199
pybind11::return_value_policy::reference, "path"_a, "description"_a,
201200
"is_executable"_a = false)
201+
.def("provideFileContent", &frontend::Frontend::provideFileContent,
202+
pybind11::return_value_policy::reference, "content"_a,
203+
"description"_a, "is_executable"_a = false)
202204
.def("addExecution", &frontend::Frontend::addExecution,
203205
pybind11::return_value_policy::reference, "description"_a)
204206
.def("addExecutionGroup", &frontend::Frontend::addExecutionGroup,

‎cpp/util/file.cpp‎

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -403,21 +403,68 @@ kj::Promise<void> next_chunk(HandleRequestFileData data) {
403403
} // namespace
404404

405405
kj::Promise<void> File::HandleRequestFile(
406-
const std::string& path, capnproto::FileReceiver::Client receiver) {
406+
FileWrapper *wrapper, capnproto::FileReceiver::Client receiver) {
407407
// TODO: see if we can easily avoid the extra round-trips while
408408
// still guaranteeing in-order processing (when capnp implements streams?)
409409
// Possibly by using UnionPromiseBuilder?
410410
if (HandleRequestFileData::num_concurrent <
411411
HandleRequestFileData::max_concurrent) {
412412
HandleRequestFileData::num_concurrent++;
413-
HandleRequestFileData data{Read(path), receiver};
413+
HandleRequestFileData data{wrapper->Read(), receiver};
414414
return next_chunk(std::move(data));
415415
}
416416
auto pf = kj::newPromiseAndFulfiller<void>();
417417
HandleRequestFileData::waiting.push(std::move(pf.fulfiller));
418-
return pf.promise.then([path, receiver]() mutable {
419-
return File::HandleRequestFile(path, receiver);
418+
return pf.promise.then([wrapper = wrapper, receiver]() mutable {
419+
return File::HandleRequestFile(wrapper, receiver);
420420
});
421421
}
422422

423+
kj::Promise<void> File::HandleRequestFile(
424+
const util::SHA256_t &hash, capnproto::FileReceiver::Client receiver) {
425+
if (hash.hasContents()) {
426+
auto req = receiver.sendChunkRequest();
427+
req.setChunk(hash.getContents());
428+
return req.send().ignoreResult().then([receiver]() mutable {
429+
return receiver.sendChunkRequest().send().ignoreResult();
430+
});
431+
}
432+
util::FileWrapper wrapper = FileWrapper::FromPath(PathForHash(hash));
433+
return HandleRequestFile(&wrapper, receiver);
434+
}
435+
436+
FileWrapper FileWrapper::FromPath(std::string path) {
437+
FileWrapper file;
438+
file.type_ = FileWrapper::FileWrapperType::PATH;
439+
file.path_ = std::move(path);
440+
return file;
441+
}
442+
443+
FileWrapper FileWrapper::FromContent(std::string content) {
444+
FileWrapper file;
445+
file.type_ = FileWrapper::FileWrapperType::CONTENT;
446+
file.content_ = std::move(content);
447+
return file;
448+
}
449+
450+
File::ChunkProducer FileWrapper::Read() {
451+
if (type_ == FileWrapper::FileWrapperType::PATH) return File::Read(path_);
452+
453+
std::unique_ptr<size_t> pos = std::make_unique<size_t>(0);
454+
return [this, pos = std::move(pos)]() mutable {
455+
if (*pos < content_.size()) {
456+
auto end =
457+
std::min(content_.begin() + *pos + util::kChunkSize,
458+
content_.end());
459+
size_t amount = end - content_.begin() - *pos;
460+
auto chunk = util::File::Chunk(
461+
// NOLINTNEXTLINE
462+
reinterpret_cast<const kj::byte *>(&content_[0] + *pos), amount);
463+
*pos += amount;
464+
return chunk;
465+
}
466+
return util::File::Chunk();
467+
};
468+
}
469+
423470
} // namespace util

‎cpp/util/file.hpp‎

Lines changed: 40 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ namespace util {
1414
static const constexpr uint32_t kChunkSize = 1024 * 1024;
1515
static const constexpr uint32_t kInlineChunkThresh = 1024;
1616

17+
class FileWrapper;
18+
1719
class File {
1820
public:
1921
// A non-owning pointer to a sequence of bytes, usually representing a part of
@@ -93,18 +95,9 @@ class File {
9395

9496
// Utility to implement RequestFile methods, given the path and the receiver
9597
static kj::Promise<void> HandleRequestFile(
96-
const std::string& path, capnproto::FileReceiver::Client receiver);
98+
FileWrapper *wrapper, capnproto::FileReceiver::Client receiver);
9799
static kj::Promise<void> HandleRequestFile(
98-
const util::SHA256_t& hash, capnproto::FileReceiver::Client receiver) {
99-
if (hash.hasContents()) {
100-
auto req = receiver.sendChunkRequest();
101-
req.setChunk(hash.getContents());
102-
return req.send().ignoreResult().then([receiver]() mutable {
103-
return receiver.sendChunkRequest().send().ignoreResult();
104-
});
105-
}
106-
return HandleRequestFile(PathForHash(hash), receiver);
107-
}
100+
const util::SHA256_t &hash, capnproto::FileReceiver::Client receiver);
108101

109102
// Utility to implement RequestFile methods
110103
template <typename RequestFileContext>
@@ -191,6 +184,42 @@ class TempDir {
191184
bool moved_ = false;
192185
};
193186

187+
// Wrapper around a File, it can be a path to a file or the content of a file.
188+
class FileWrapper {
189+
enum class FileWrapperType {
190+
PATH = 0, CONTENT = 1
191+
};
192+
193+
FileWrapper() = default;
194+
KJ_DISALLOW_COPY(FileWrapper);
195+
196+
public:
197+
~FileWrapper() = default;
198+
199+
FileWrapper(FileWrapper &&other) noexcept { *this = std::move(other); }
200+
201+
FileWrapper &operator=(FileWrapper &&other) noexcept {
202+
path_ = std::move(other.path_);
203+
content_ = std::move(other.content_);
204+
type_ = other.type_;
205+
return *this;
206+
}
207+
208+
// Create a wrapper from a file path
209+
static FileWrapper FromPath(std::string path);
210+
211+
// Create a wrapper from a file content
212+
static FileWrapper FromContent(std::string content);
213+
214+
// Returns a ChunkProducer with the content of the wrapped file
215+
File::ChunkProducer Read();
216+
217+
private:
218+
std::string path_;
219+
std::string content_;
220+
FileWrapperType type_{};
221+
};
222+
194223
} // namespace util
195224

196225
#endif

‎python/CMakeLists.txt‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Sources
22
file(GLOB SOURCES_PY "${CMAKE_CURRENT_SOURCE_DIR}/*.py")
3-
set(PY_MODULES formats tests uis languages sanity_checks)
3+
set(PY_MODULES formats tests uis languages sanity_checks statements)
44
foreach(module_name ${PY_MODULES})
55
file(GLOB SOURCES_PY_${module_name}
66
"${CMAKE_CURRENT_SOURCE_DIR}/${module_name}/*.py")
@@ -63,6 +63,15 @@ foreach(module_name ${PY_MODULES})
6363
DEPENDS "${SOURCES_PY_${module_name}}")
6464
endforeach()
6565

66+
# copy the statement templates
67+
set(STATEMENTS_TEMPLATES_SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/statements/templates")
68+
set(STATEMENTS_TEMPLATES_DST_DIR "${SOURCES_statements_MODULE}/templates")
69+
add_custom_command(OUTPUT ${STATEMENTS_TEMPLATES_DST_DIR}
70+
COMMAND ${CMAKE_COMMAND} -E copy_directory
71+
${STATEMENTS_TEMPLATES_SRC_DIR}
72+
${STATEMENTS_TEMPLATES_DST_DIR}
73+
DEPENDS ${STATEMENTS_TEMPLATES_SRC_DIR})
74+
6675
add_custom_command(OUTPUT ${REQUIREMENTS_TXT_TGT}
6776
COMMAND ${CMAKE_COMMAND} -E copy ${REQUIREMENTS_TXT}
6877
${REQUIREMENTS_TXT_TGT}
@@ -128,6 +137,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/version.py.in ${VERSION_PY})
128137

129138
add_custom_target(module_files ALL
130139
DEPENDS ${MODULES_TGT}
140+
${STATEMENTS_TEMPLATES_DST_DIR}
131141
${REQUIREMENTS_TXT_TGT}
132142
"${TESTS_TASKS_TGT}"
133143
${SOURCES_PY_TGT}

‎python/args.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,18 @@ def add_terry_group(parser: argparse.ArgumentParser):
242242
action="store")
243243

244244

245+
def add_statement_group(parser: argparse.ArgumentParser):
246+
group = parser.add_argument_group("Statement options")
247+
group.add_argument(
248+
"--no-statement",
249+
help="Do not compile the statements",
250+
action="store_true")
251+
group.add_argument(
252+
"--set",
253+
help="Set a template variable for the statements",
254+
action="append")
255+
256+
245257
def add_help_group(parser: argparse.ArgumentParser):
246258
group = parser.add_argument_group("Help options")
247259
group.add_argument(
@@ -270,6 +282,7 @@ def get_parser(bulk: bool) -> argparse.ArgumentParser:
270282
add_execution_group(parser)
271283
add_ioi_group(parser)
272284
add_terry_group(parser)
285+
add_statement_group(parser)
273286
add_help_group(parser)
274287
if bulk:
275288
add_bulk_group(parser)

‎python/config.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ class Config:
2727
"execution": ["exclusive", "extra_time", "copy_exe"],
2828
"ioi": ["detailed_checker"],
2929
"terry": ["arch", "seed"],
30+
"statement": ["no_statement", "set"],
3031
"help": ["help_colors"],
3132
"bulk": ["contest_dir", "contest_yaml"]
3233
}
@@ -40,7 +41,8 @@ class Config:
4041
def __init__(self):
4142
# generic group
4243
self.solutions = [] # type: List[str]
43-
self.task_dir = os.getcwd()
44+
# $PWD for tasks inside symlinks
45+
self.task_dir = os.getenv("PWD", os.getcwd())
4446
self.max_depth = 2
4547
self.ui = UIS.CURSES
4648
self.cache = CacheMode.ALL
@@ -89,6 +91,10 @@ def __init__(self):
8991
self.arch = Arch.DEFAULT
9092
self.seed = None
9193

94+
# statement group
95+
self.no_statement = False
96+
self.set = []
97+
9298
# help group
9399
self.help_colors = False
94100

‎python/formats/__init__.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ def __init__(self, name: str, title: str, subtasks: Dict[int, Subtask],
219219
grader_map: Dict[Language, GraderInfo],
220220
checker: Optional["SourceFile"], time_limit: float,
221221
memory_limit_kb: int, input_file: str, output_file: str,
222-
task_type: TaskType):
222+
task_type: TaskType, yaml: Any):
223223
super().__init__(name, title)
224224
self.subtasks = subtasks
225225
self.official_solution = official_solution
@@ -230,6 +230,7 @@ def __init__(self, name: str, title: str, subtasks: Dict[int, Subtask],
230230
self.input_file = input_file
231231
self.output_file = output_file
232232
self.task_type = task_type
233+
self.yaml = yaml
233234

234235
self.default_gen = None # type: Optional[Generator]
235236
self.default_val = None # type: Optional[Validator]

0 commit comments

Comments
 (0)