In commit "ipc: hold event loop references when using it in CapnpProtocol" (2d001df8a60a82c286ca46b04de2917be4a5c74f)
There is a bug in the code I suggested: done() returns m_num_refs == 0 && m_async_fns->empty(), and m_async_fns is only set while loop() runs: it is emplaced when loop() starts and reset before it returns. In serve(), after the parent disconnects and loop() returns, there is a window before serve() resets m_loop where a makeStream() or disconnectIncoming() call from another thread gets past the !m_loop check and calls done() with no references and m_async_fns unset. That is undefined behavior, and with _GLIBCXX_ASSERTIONS it should abort.
This can be avoided by having serve() hold a reference to the loop, the way startLoop() does with m_loop_ref, and release it when the parent disconnects:
auto [server, it] = mp::ServeStream<messages::Init>(*loop, make_stream(), init);
it->onDisconnect([this]() EXCLUSIVE_LOCKS_REQUIRED(!m_loop_mutex) {
LOCK(m_loop_mutex);
m_loop_ref.reset();
});
Then loopRef() can assume the loop is running and just add a reference:
[[nodiscard]] std::optional<mp::EventLoopRef> loopRef() EXCLUSIVE_LOCKS_REQUIRED(!m_loop_mutex)
{
LOCK(m_loop_mutex);
if (!m_loop_ref) return std::nullopt;
return mp::EventLoopRef{**m_loop_ref};
}
This drops the done() call and the try/catch in disconnectIncoming(), which becomes if (!ref) return;. It also drops setup_ref, which was only needed so serve() could call loopRef() before loop() started. m_loop_ref becomes GUARDED_BY(m_loop_mutex) and is set in the same locked block as m_loop in both serve() and startLoop(), and connect(), listen() and makeStream() throw "IPC event loop is not running" when loopRef() returns nullopt.
I tried this on top of 2d001df8, and ipc_tests and the IPC functional tests pass, including rpc_misc.py, whose echoipc spawns a child process that runs serve(). Full diff:
<details><summary>Diff</summary>
@@ -76,20 +76,20 @@ public:
CapnpProtocol(const char* exe_name) : m_exe_name{exe_name} {}
~CapnpProtocol() noexcept(true)
{
- m_loop_ref.reset();
+ WITH_LOCK(m_loop_mutex, m_loop_ref.reset());
if (m_loop_thread.joinable()) m_loop_thread.join();
assert(WITH_LOCK(m_loop_mutex, return !m_loop));
};
std::unique_ptr<interfaces::Init> connect(mp::Stream stream) override EXCLUSIVE_LOCKS_REQUIRED(!m_loop_mutex)
{
startLoop();
- mp::EventLoopRef ref{loopRef()};
+ mp::EventLoopRef ref{requireLoopRef()};
return mp::ConnectStream<messages::Init>(*ref, std::move(stream));
}
void listen(mp::SocketId listen_fd, interfaces::Init& init, std::optional<size_t> max_connections) override EXCLUSIVE_LOCKS_REQUIRED(!m_loop_mutex)
{
startLoop();
- mp::EventLoopRef ref{loopRef()};
+ mp::EventLoopRef ref{requireLoopRef()};
if (::listen(listen_fd, /*backlog=*/5) != 0) {
throw std::system_error(errno, std::system_category());
}
@@ -107,31 +107,27 @@ public:
LOCK(m_loop_mutex);
assert(!m_loop);
loop = &m_loop.emplace(m_exe_name, std::move(opts), &m_context);
+ m_loop_ref.emplace(*loop);
}
- {
- // Hold a reference while setting up the parent connection.
- // The make_stream() callback can call makeStream(), which calls
- // loopRef(), and EventLoop::done() at this point will wrongly return
- // true, making loopRef() throw the "IPC event loop is shutting down"
- // error.
- mp::EventLoopRef setup_ref{*loop};
- mp::ServeStream<messages::Init>(*loop, make_stream(), init);
- }
- m_parent_connection = &loop->m_incoming_connections.back();
+ // Release m_loop_ref when the parent disconnects, so loopRef() stops
+ // handing out new references and the loop exits once existing ones
+ // are released.
+ auto [server, it] = mp::ServeStream<messages::Init>(*loop, make_stream(), init);
+ it->onDisconnect([this]() EXCLUSIVE_LOCKS_REQUIRED(!m_loop_mutex) {
+ LOCK(m_loop_mutex);
+ m_loop_ref.reset();
+ });
+ m_parent_connection = &*it;
loop->loop();
LOCK(m_loop_mutex);
m_loop.reset();
}
void disconnectIncoming() override EXCLUSIVE_LOCKS_REQUIRED(!m_loop_mutex)
{
- // Return instead of throwing if the loop is not running, since this is
- // called during shutdown.
- std::optional<mp::EventLoopRef> ref;
- try {
- ref.emplace(loopRef());
- } catch (const Exception&) {
- return;
- }
+ // Do nothing if the loop is not running, since this is called during
+ // shutdown.
+ auto ref{loopRef()};
+ if (!ref) return;
// Delete incoming connections, except the connection to a parent
// process (if there is one), since a parent process should be able to
// monitor and control this process, even during shutdown.
@@ -142,7 +138,7 @@ public:
mp::Stream makeStream(mp::SocketId socket) override EXCLUSIVE_LOCKS_REQUIRED(!m_loop_mutex)
{
startLoop();
- mp::EventLoopRef ref{loopRef()};
+ mp::EventLoopRef ref{requireLoopRef()};
return mp::MakeStream(*ref, socket);
}
void addCleanup(std::type_index type, void* iface, std::function<void()> cleanup) override
@@ -164,8 +160,8 @@ public:
{
LOCK(m_loop_mutex);
loop = &m_loop.emplace(m_exe_name, std::move(opts), &m_context);
+ m_loop_ref.emplace(*loop);
}
- m_loop_ref.emplace(*loop);
promise.set_value();
loop->loop();
LOCK(m_loop_mutex);
@@ -173,30 +169,35 @@ public:
});
promise.get_future().wait();
}
- //! Return a reference that keeps the event loop running, or throw if the
- //! loop is not running. Threads that are not handling an IPC request need
- //! this, because in serve() mode nothing else keeps the loop running for
- //! them.
- [[nodiscard]] mp::EventLoopRef loopRef() EXCLUSIVE_LOCKS_REQUIRED(!m_loop_mutex)
+ //! Return a reference that keeps the event loop running, or nullopt if the
+ //! loop is not running or, in serve() mode, the parent process has
+ //! disconnected. Threads that are not handling an IPC request need this,
+ //! because in serve() mode nothing else keeps the loop running for them.
+ [[nodiscard]] std::optional<mp::EventLoopRef> loopRef() EXCLUSIVE_LOCKS_REQUIRED(!m_loop_mutex)
{
LOCK(m_loop_mutex);
- if (!m_loop) throw Exception("IPC event loop is not running");
- mp::Lock lock(m_loop->m_mutex);
- if (m_loop->done()) throw Exception("IPC event loop is shutting down");
- mp::EventLoopRef ref{*m_loop, &lock};
- ref.m_lock = nullptr; // Lock m_mutex again when the reference is released.
- return ref;
+ if (!m_loop_ref) return std::nullopt;
+ return mp::EventLoopRef{**m_loop_ref};
+ }
+ [[nodiscard]] mp::EventLoopRef requireLoopRef() EXCLUSIVE_LOCKS_REQUIRED(!m_loop_mutex)
+ {
+ auto ref{loopRef()};
+ if (!ref) throw Exception("IPC event loop is not running");
+ return std::move(*ref);
}
const char* m_exe_name;
Context m_context;
- //! Guards m_loop.
+ //! Guards m_loop and m_loop_ref.
Mutex m_loop_mutex;
//! EventLoop object which manages I/O events for all connections.
std::optional<mp::EventLoop> m_loop GUARDED_BY(m_loop_mutex);
- //! Reference to the same EventLoop. Increments the loop’s refcount on
- //! creation, decrements on destruction. The loop thread exits when the
- //! refcount reaches 0. Other IPC objects also hold their own EventLoopRef.
- std::optional<mp::EventLoopRef> m_loop_ref;
+ //! Reference to the same EventLoop, set whenever the loop is running and
+ //! accepting new work. While it is set the loop can't exit, so loopRef()
+ //! can add references without checking EventLoop::done(). Released in the
+ //! destructor in startLoop() mode, and when the parent disconnects in
+ //! serve() mode. The loop exits when the refcount reaches 0. Other IPC
+ //! objects also hold their own EventLoopRef.
+ std::optional<mp::EventLoopRef> m_loop_ref GUARDED_BY(m_loop_mutex);
//! Connection to parent, if this is a child process spawned by a parent process.
mp::Connection* m_parent_connection{nullptr};
std::thread m_loop_thread;
</details>