fix/kpn-wedging-audit #3
@@ -220,6 +220,11 @@ struct ResourceSnapshot {
|
|||||||
struct IResourceProbe {
|
struct IResourceProbe {
|
||||||
virtual ~IResourceProbe() = default;
|
virtual ~IResourceProbe() = default;
|
||||||
virtual ResourceSnapshot snapshot(const std::string& name) const = 0;
|
virtual ResourceSnapshot snapshot(const std::string& name) const = 0;
|
||||||
|
|
||||||
|
/// Release every thread waiting for the resource, so teardown is not held
|
||||||
|
/// up by one. A network calls this on the resources registered with it when
|
||||||
|
/// it halts; default no-op for probes with nothing to wake.
|
||||||
|
virtual void close() {}
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace kpn
|
} // namespace kpn
|
||||||
|
|||||||
@@ -14,6 +14,18 @@ namespace kpn {
|
|||||||
|
|
||||||
template<typename T> class Channel; // forward declaration for acquire_balanced
|
template<typename T> class Channel; // forward declaration for acquire_balanced
|
||||||
|
|
||||||
|
/// Thrown by a pending acquire() when the resource is closed underneath it.
|
||||||
|
///
|
||||||
|
/// acquire() blocks on a condition variable with no timeout and no stop
|
||||||
|
/// condition, so a node parked there ignored teardown entirely: the worker
|
||||||
|
/// never returned, the pool's join never completed, and shutdown hung on a
|
||||||
|
/// resource nobody was going to release. Closing the resource turns that into
|
||||||
|
/// an exception the node's normal error path already handles.
|
||||||
|
class ResourceClosedError : public std::runtime_error {
|
||||||
|
public:
|
||||||
|
ResourceClosedError() : std::runtime_error("shared resource closed") {}
|
||||||
|
};
|
||||||
|
|
||||||
// ── SharedResource ────────────────────────────────────────────────────────────
|
// ── SharedResource ────────────────────────────────────────────────────────────
|
||||||
//
|
//
|
||||||
// Wraps an exclusive resource (e.g. an ONNX session, a CUDA stream) and
|
// Wraps an exclusive resource (e.g. an ONNX session, a CUDA stream) and
|
||||||
@@ -72,6 +84,7 @@ public:
|
|||||||
template<typename PriorityFn>
|
template<typename PriorityFn>
|
||||||
Guard acquire(PriorityFn&& fn) {
|
Guard acquire(PriorityFn&& fn) {
|
||||||
std::unique_lock lock(mutex_);
|
std::unique_lock lock(mutex_);
|
||||||
|
if (closed_) throw ResourceClosedError{};
|
||||||
if (!held_) {
|
if (!held_) {
|
||||||
held_ = true;
|
held_ = true;
|
||||||
acq_.fetch_add(1, std::memory_order_relaxed);
|
acq_.fetch_add(1, std::memory_order_relaxed);
|
||||||
@@ -83,18 +96,46 @@ public:
|
|||||||
current_waiters_.store(waiters_.size(), std::memory_order_relaxed);
|
current_waiters_.store(waiters_.size(), std::memory_order_relaxed);
|
||||||
|
|
||||||
auto t0 = w.wait_start;
|
auto t0 = w.wait_start;
|
||||||
w.cv.wait(lock, [&w] { return w.ready; });
|
// Woken either by release() handing over ownership, or by close()
|
||||||
|
// giving up on the wait entirely.
|
||||||
|
w.cv.wait(lock, [&w] { return w.ready || w.closed; });
|
||||||
|
|
||||||
int64_t wait_us = std::chrono::duration_cast<std::chrono::microseconds>(
|
int64_t wait_us = std::chrono::duration_cast<std::chrono::microseconds>(
|
||||||
clock_t::now() - t0).count();
|
clock_t::now() - t0).count();
|
||||||
waiters_.erase(std::find(waiters_.begin(), waiters_.end(), &w));
|
waiters_.erase(std::find(waiters_.begin(), waiters_.end(), &w));
|
||||||
current_waiters_.store(waiters_.size(), std::memory_order_relaxed);
|
current_waiters_.store(waiters_.size(), std::memory_order_relaxed);
|
||||||
acq_.fetch_add(1, std::memory_order_relaxed);
|
|
||||||
total_wait_us_.fetch_add(static_cast<uint64_t>(wait_us > 0 ? wait_us : 0),
|
total_wait_us_.fetch_add(static_cast<uint64_t>(wait_us > 0 ? wait_us : 0),
|
||||||
std::memory_order_relaxed);
|
std::memory_order_relaxed);
|
||||||
|
|
||||||
|
// Closed without being handed ownership: no Guard, so nothing to
|
||||||
|
// release, and held_ is left exactly as close() found it.
|
||||||
|
if (!w.ready) throw ResourceClosedError{};
|
||||||
|
|
||||||
|
acq_.fetch_add(1, std::memory_order_relaxed);
|
||||||
return Guard(this);
|
return Guard(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Wake every waiter and refuse further acquisitions.
|
||||||
|
///
|
||||||
|
/// Teardown is the whole point: a node parked in acquire() is not
|
||||||
|
/// observing stop flags, so without this the only way out is for whoever
|
||||||
|
/// holds the resource to release it — which, if that node is also being
|
||||||
|
/// stopped, may never happen. Idempotent, and safe to call from any thread.
|
||||||
|
void close() override {
|
||||||
|
std::lock_guard lock(mutex_);
|
||||||
|
closed_ = true;
|
||||||
|
for (Waiter* w : waiters_) {
|
||||||
|
w->closed = true;
|
||||||
|
w->cv.notify_one();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reopen after a close(). For reuse across runs; not needed for teardown.
|
||||||
|
void reopen() {
|
||||||
|
std::lock_guard lock(mutex_);
|
||||||
|
closed_ = false;
|
||||||
|
}
|
||||||
|
|
||||||
// Acquire with no priority (all waiters treated equally, order is fair-ish).
|
// Acquire with no priority (all waiters treated equally, order is fair-ish).
|
||||||
Guard acquire() {
|
Guard acquire() {
|
||||||
return acquire([] { return 0.5f; });
|
return acquire([] { return 0.5f; });
|
||||||
@@ -132,7 +173,10 @@ public:
|
|||||||
private:
|
private:
|
||||||
void release() {
|
void release() {
|
||||||
std::unique_lock lock(mutex_);
|
std::unique_lock lock(mutex_);
|
||||||
if (waiters_.empty()) {
|
// Hand over only to a waiter that is still waiting. A closed one is on
|
||||||
|
// its way out and will not take ownership, so treating it as the next
|
||||||
|
// holder would leave held_ true with nobody holding it.
|
||||||
|
if (closed_ || waiters_.empty()) {
|
||||||
held_ = false;
|
held_ = false;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -162,7 +206,8 @@ private:
|
|||||||
std::function<float()> priority_fn;
|
std::function<float()> priority_fn;
|
||||||
clock_t::time_point wait_start;
|
clock_t::time_point wait_start;
|
||||||
std::condition_variable cv;
|
std::condition_variable cv;
|
||||||
bool ready{false};
|
bool ready{false}; // handed ownership by release()
|
||||||
|
bool closed{false}; // woken by close() instead
|
||||||
|
|
||||||
Waiter(std::function<float()> fn, clock_t::time_point t)
|
Waiter(std::function<float()> fn, clock_t::time_point t)
|
||||||
: priority_fn(std::move(fn)), wait_start(t) {}
|
: priority_fn(std::move(fn)), wait_start(t) {}
|
||||||
@@ -172,6 +217,7 @@ private:
|
|||||||
|
|
||||||
T resource_;
|
T resource_;
|
||||||
bool held_{false};
|
bool held_{false};
|
||||||
|
bool closed_{false};
|
||||||
mutable std::mutex mutex_;
|
mutable std::mutex mutex_;
|
||||||
std::vector<Waiter*> waiters_;
|
std::vector<Waiter*> waiters_;
|
||||||
std::atomic<uint64_t> acq_{0};
|
std::atomic<uint64_t> acq_{0};
|
||||||
|
|||||||
@@ -160,6 +160,11 @@ public:
|
|||||||
#ifdef KPN_WEB_DEBUG
|
#ifdef KPN_WEB_DEBUG
|
||||||
if (web_server_) web_server_->stop();
|
if (web_server_) web_server_->stop();
|
||||||
#endif
|
#endif
|
||||||
|
// Release anything parked on a shared resource first. A node blocked in
|
||||||
|
// acquire() is not watching stop flags, so stopping it would wait on a
|
||||||
|
// handover that may never come — its holder is being stopped too.
|
||||||
|
for (auto& [rname, probe] : resource_probes_) { (void)rname; probe->close(); }
|
||||||
|
|
||||||
for (auto it = fanout_nodes_ptr_.rbegin(); it != fanout_nodes_ptr_.rend(); ++it)
|
for (auto it = fanout_nodes_ptr_.rbegin(); it != fanout_nodes_ptr_.rend(); ++it)
|
||||||
(*it)->stop();
|
(*it)->stop();
|
||||||
for (auto it = user_nodes_topo_.rbegin(); it != user_nodes_topo_.rend(); ++it)
|
for (auto it = user_nodes_topo_.rbegin(); it != user_nodes_topo_.rend(); ++it)
|
||||||
@@ -173,6 +178,7 @@ public:
|
|||||||
#ifdef KPN_WEB_DEBUG
|
#ifdef KPN_WEB_DEBUG
|
||||||
if (web_server_) web_server_->stop();
|
if (web_server_) web_server_->stop();
|
||||||
#endif
|
#endif
|
||||||
|
for (auto& [rname, probe] : resource_probes_) { (void)rname; probe->close(); }
|
||||||
// user_nodes_topo_ is already in sources-first order.
|
// user_nodes_topo_ is already in sources-first order.
|
||||||
// Stop each node and drain its output channels before moving on.
|
// Stop each node and drain its output channels before moving on.
|
||||||
for (std::size_t i = 0; i < user_nodes_topo_.size(); ++i) {
|
for (std::size_t i = 0; i < user_nodes_topo_.size(); ++i) {
|
||||||
|
|||||||
@@ -237,3 +237,51 @@ TEST_CASE("make_shared_resource constructs with forwarded args", "[shared_resour
|
|||||||
auto g = res.acquire();
|
auto g = res.acquire();
|
||||||
REQUIRE(*g == "hello");
|
REQUIRE(*g == "hello");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Regression: a waiter must be releasable, or teardown waits on it forever.
|
||||||
|
//
|
||||||
|
// acquire() blocks on a condition variable whose predicate only becomes true
|
||||||
|
// when release() hands over ownership. There was no timeout and no stop
|
||||||
|
// condition, so a node parked there ignored teardown entirely: its worker never
|
||||||
|
// returned, the pool's join never completed, and shutdown hung waiting for a
|
||||||
|
// resource nobody was going to release — which is exactly the case when the
|
||||||
|
// holder is being stopped too.
|
||||||
|
//
|
||||||
|
// close() turns that into an exception the node's existing error path already
|
||||||
|
// handles, and networks now call it on registered resources before stopping any
|
||||||
|
// node, for the same reason.
|
||||||
|
TEST_CASE("closing a shared resource releases its waiters", "[shared_resource]") {
|
||||||
|
SharedResource<int> res(42);
|
||||||
|
|
||||||
|
auto holder = res.acquire(); // resource is now held
|
||||||
|
|
||||||
|
std::atomic<bool> threw{false}, returned{false};
|
||||||
|
std::thread waiter([&] {
|
||||||
|
try {
|
||||||
|
auto g = res.acquire(); // blocks: someone else holds it
|
||||||
|
(void)g;
|
||||||
|
} catch (const ResourceClosedError&) {
|
||||||
|
threw.store(true, std::memory_order_release);
|
||||||
|
}
|
||||||
|
returned.store(true, std::memory_order_release);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Let it park, then tear down without ever releasing the holder.
|
||||||
|
std::this_thread::sleep_for(std::chrono::milliseconds(50));
|
||||||
|
REQUIRE_FALSE(returned.load(std::memory_order_acquire));
|
||||||
|
|
||||||
|
res.close();
|
||||||
|
waiter.join();
|
||||||
|
|
||||||
|
CHECK(threw.load(std::memory_order_acquire));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_CASE("acquiring a closed resource fails immediately", "[shared_resource]") {
|
||||||
|
SharedResource<int> res(7);
|
||||||
|
res.close();
|
||||||
|
CHECK_THROWS_AS(res.acquire(), ResourceClosedError);
|
||||||
|
|
||||||
|
// Reusable across runs once reopened.
|
||||||
|
res.reopen();
|
||||||
|
CHECK_NOTHROW(res.acquire());
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user