Refactor Subzero initialization and add a browser callback handler.
Handlers are represented as a "compile server" even though
right now it can really only handle a single
compile request.
Then there can be a commandline-based server and a
browser-based server. This server takes over the main
thread. In the browser-based case the server can block,
waiting on bytes to be pushed. This becomes a producer of
bitcode bytes.
The original main thread which did bitcode reading is now
shifted to yet another worker thread, which is then the
consumer of bitcode bytes.
This uses an IRT interface for listening to messages
from the browser:
https://codereview.chromium.org/984713003/
TEST=Build the IRT core nexe w/ the above patch and compile w/ something like:
echo """
readwrite_file objfile /tmp/temp.nexe---gcc.opt.stripped.pexe---.o
rpc StreamInitWithSplit i(4) h(objfile) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) C(4,-O2\x00) * s()
stream_file /usr/local/google/home/jvoung/pexe_tests/gcc.opt.stripped.pexe 65536 1000000000
rpc StreamEnd * i() s() s() s()
echo "pnacl-sz complete"
""" | scons-out/opt-linux-x86-32/staging/sel_universal \
-a -B scons-out/nacl_irt-x86-32/staging/irt_core.nexe \
--abort_on_error \
-- toolchain/linux_x86/pnacl_translator/translator/x86-32/bin/pnacl-sz.nexe
echo """
readwrite_file nexefile /tmp/temp.nexe.tmp
readonly_file objfile0 /tmp/temp.nexe---gcc.opt.stripped.pexe---.o
rpc RunWithSplit i(1) h(objfile0) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(invalid) h(nexefile) *
echo "ld complete"
""" | /usr/local/google/home/nacl3/native_client/scons-out/opt-linux-x86-32/staging/sel_universal \
--abort_on_error \
-a -B \
scons-out/nacl_irt-x86-32/staging/irt_core.nexe \
-E NACL_IRT_OPEN_RESOURCE_BASE=toolchain/linux_x86/pnacl_translator/translator/x86-32/lib/ \
-E NACL_IRT_OPEN_RESOURCE_REMAP=libpnacl_irt_shim.a:libpnacl_irt_shim_dummy.a \
-- toolchain/linux_x86/pnacl_translator/translator/x86-32/bin/ld.nexe
BUG= https://code.google.com/p/nativeclient/issues/detail?id=4091
R=kschimpf@google.com, stichnot@chromium.org
Review URL: https://codereview.chromium.org/997773002
diff --git a/src/IceBrowserCompileServer.cpp b/src/IceBrowserCompileServer.cpp
new file mode 100644
index 0000000..6a9ee65
--- /dev/null
+++ b/src/IceBrowserCompileServer.cpp
@@ -0,0 +1,150 @@
+//===- subzero/src/IceBrowserCompileServer.cpp - Browser compile server ---===//
+//
+// The Subzero Code Generator
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the browser-based compile server.
+//
+//===----------------------------------------------------------------------===//
+
+// Can only compile this with the NaCl compiler (needs irt.h, and the
+// unsandboxed LLVM build using the trusted compiler does not have irt.h).
+#if PNACL_BROWSER_TRANSLATOR
+
+#include <cstring>
+#include <irt.h>
+#include <irt_dev.h>
+#include <thread>
+
+#include "llvm/Support/QueueStreamer.h"
+
+#include "IceBrowserCompileServer.h"
+
+namespace Ice {
+
+// Create C wrappers around callback handlers for the IRT interface.
+namespace {
+
+BrowserCompileServer *gCompileServer;
+struct nacl_irt_private_pnacl_translator_compile gIRTFuncs;
+
+void getIRTInterfaces() {
+ size_t QueryResult =
+ nacl_interface_query(NACL_IRT_PRIVATE_PNACL_TRANSLATOR_COMPILE_v0_1,
+ &gIRTFuncs, sizeof(gIRTFuncs));
+ if (QueryResult != sizeof(gIRTFuncs))
+ llvm::report_fatal_error("Failed to get translator compile IRT interface");
+}
+
+char *onInitCallback(uint32_t NumThreads, int *ObjFileFDs,
+ size_t ObjFileFDCount, char **argv, size_t argc) {
+ if (ObjFileFDCount < 1) {
+ std::string Buffer;
+ llvm::raw_string_ostream StrBuf(Buffer);
+ StrBuf << "Invalid number of FDs for onInitCallback " << ObjFileFDCount
+ << "\n";
+ return strdup(StrBuf.str().c_str());
+ }
+ int ObjFileFD = ObjFileFDs[0];
+ if (ObjFileFD < 0) {
+ std::string Buffer;
+ llvm::raw_string_ostream StrBuf(Buffer);
+ StrBuf << "Invalid FD given for onInitCallback " << ObjFileFD << "\n";
+ return strdup(StrBuf.str().c_str());
+ }
+ // NOTE: argv is owned by the caller, but we parse here before returning.
+ gCompileServer->getParsedFlags(NumThreads, argc, argv);
+ gCompileServer->startCompileThread(ObjFileFD);
+ return nullptr;
+}
+
+int onDataCallback(const void *Data, size_t NumBytes) {
+ return gCompileServer->pushInputBytes(Data, NumBytes) ? 1 : 0;
+}
+
+char *onEndCallback() {
+ gCompileServer->endInputStream();
+ gCompileServer->waitForCompileThread();
+ // TODO(jvoung): Also return an error string, and UMA data.
+ // Set up a report_fatal_error handler to grab that string.
+ if (gCompileServer->getErrorCode().value()) {
+ return strdup("Some error occurred");
+ }
+ return nullptr;
+}
+
+struct nacl_irt_pnacl_compile_funcs SubzeroCallbacks {
+ &onInitCallback, &onDataCallback, &onEndCallback
+};
+
+std::unique_ptr<llvm::raw_fd_ostream> getOutputStream(int FD) {
+ if (FD <= 0)
+ llvm::report_fatal_error("Invalid output FD");
+ const bool CloseOnDtor = true;
+ const bool Unbuffered = false;
+ return std::unique_ptr<llvm::raw_fd_ostream>(
+ new llvm::raw_fd_ostream(FD, CloseOnDtor, Unbuffered));
+}
+
+} // end of anonymous namespace
+
+BrowserCompileServer::~BrowserCompileServer() {}
+
+void BrowserCompileServer::run() {
+ gCompileServer = this;
+ // TODO(jvoung): make a useful fatal error handler that can
+ // exit all *worker* threads, keep the server thread alive,
+ // and be able to handle a server request for the last error string.
+ getIRTInterfaces();
+ gIRTFuncs.serve_translate_request(&SubzeroCallbacks);
+}
+
+void BrowserCompileServer::getParsedFlags(uint32_t NumThreads, int argc,
+ char **argv) {
+ ClFlags::parseFlags(argc, argv);
+ ClFlags::getParsedClFlags(Flags);
+ ClFlags::getParsedClFlagsExtra(ExtraFlags);
+ // Set some defaults which aren't specified via the argv string.
+ Flags.setNumTranslationThreads(NumThreads);
+ Flags.setUseSandboxing(true);
+ Flags.setOutFileType(FT_Elf);
+ // TODO(jvoung): allow setting different target arches.
+ Flags.setTargetArch(Target_X8632);
+ ExtraFlags.setBuildOnRead(true);
+ ExtraFlags.setInputFileFormat(llvm::PNaClFormat);
+}
+
+bool BrowserCompileServer::pushInputBytes(const void *Data, size_t NumBytes) {
+ return InputStream->PutBytes(
+ const_cast<unsigned char *>(
+ reinterpret_cast<const unsigned char *>(Data)),
+ NumBytes) != NumBytes;
+}
+
+void BrowserCompileServer::endInputStream() { InputStream->SetDone(); }
+
+void BrowserCompileServer::startCompileThread(int ObjFD) {
+ InputStream = new llvm::QueueStreamer();
+ LogStream = getOutputStream(STDOUT_FILENO);
+ LogStream->SetUnbuffered();
+ EmitStream = getOutputStream(ObjFD);
+ EmitStream->SetBufferSize(1 << 14);
+ ELFStream.reset(new ELFStreamer(*EmitStream.get()));
+ Ctx.reset(new GlobalContext(LogStream.get(), EmitStream.get(),
+ ELFStream.get(), Flags));
+ CompileThread = std::thread([this]() {
+ Ctx->initParserThread();
+ this->getCompiler().run(ExtraFlags, *Ctx.get(),
+ // Retain original reference, but the compiler
+ // (LLVM's MemoryObject) wants to handle deletion.
+ std::unique_ptr<llvm::DataStreamer>(InputStream));
+ });
+}
+
+} // end of namespace Ice
+
+#endif // PNACL_BROWSER_TRANSLATOR
