Squashed 'third_party/marl/' changes from 246091e81..a047dd0bb
a047dd0bb Include benchmarks on project's README.md
4c702da52 Scheduler: Fix lock state on Fiber::wait timeout.
b4e305525 Docs: Add documentation for marl::Scheduler.
5f18ac0e0 ConditionVariable: Delete copy and move constructors
f78eb441f Scheduler: document requirement to unbind() before destruction
9f9f6d32e Defer benchmark - avoid benchmark::DoNotOptimize()
3b610e902 Fix compiler warnings with MARL_FIBERS_USE_UCONTEXT
0dbab1184 Scheduler: Delete copy and move constructors / assignment ops.
cbef55d58 Kokoro: Build benchmarks
e923c3d96 Rework the 'hello task' example to be more idiomatic
git-subtree-dir: third_party/marl
git-subtree-split: a047dd0bbbd6a65ee4d03d0ceb4fedfa56da02a5
diff --git a/README.md b/README.md
index da1f5a1..3fd10b4 100644
--- a/README.md
+++ b/README.md
@@ -18,6 +18,7 @@
#include "marl/defer.h"
#include "marl/event.h"
#include "marl/scheduler.h"
+#include "marl/waitgroup.h"
#include <cstdio>
@@ -29,33 +30,38 @@
scheduler.setWorkerThreadCount(4);
defer(scheduler.unbind()); // Automatically unbind before returning.
- // Create an event that automatically resets itself.
- marl::Event sayHellow(marl::Event::Mode::Auto);
- marl::Event saidHellow(marl::Event::Mode::Auto);
+ constexpr int numTasks = 10;
+
+ // Create an event that is manually reset.
+ marl::Event sayHellow(marl::Event::Mode::Manual);
+
+ // Create a WaitGroup with an initial count of numTasks.
+ marl::WaitGroup saidHellow(numTasks);
// Schedule some tasks to run asynchronously.
- for (int i = 0; i < 10; i++) {
+ for (int i = 0; i < numTasks; i++) {
// Each task will run on one of the 4 worker threads.
marl::schedule([=] { // All marl primitives are capture-by-value.
- printf("Task %d waiting to say hello!\n", i);
+ // Decrement the WaitGroup counter when the task has finished.
+ defer(saidHellow.done());
+
+ printf("Task %d waiting to say hello...\n", i);
// Blocking in a task?
// The scheduler will find something else for this thread to do.
sayHellow.wait();
printf("Hello from task %d!\n", i);
-
- saidHellow.signal();
});
}
- // Unblock the tasks one by one.
- for (int i = 0; i < 10; i++) {
- sayHellow.signal();
- saidHellow.wait();
- }
+ sayHellow.signal(); // Unblock all the tasks.
- // All tasks are guaranteed to completed before the scheduler is destructed.
+ saidHellow.wait(); // Wait for all tasks to complete.
+
+ printf("All tasks said hello.\n");
+
+ // All tasks are guaranteed to complete before the scheduler is destructed.
}
```
@@ -111,6 +117,10 @@
add_subdirectory(${MARL_DIR})
```
+## Benchmarks
+
+Graphs of several microbenchmarks can be found [here](https://google.github.io/marl/benchmarks).
+
---
Note: This is not an officially supported Google product
diff --git a/docs/imgs/worker_run_mtw.svg b/docs/imgs/worker_run_mtw.svg
new file mode 100644
index 0000000..3b9625f
--- /dev/null
+++ b/docs/imgs/worker_run_mtw.svg
@@ -0,0 +1,136 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="724 1659.5 441.5 408.5" width="441.5" height="408.5">
+ <defs>
+ <font-face font-family="Courier New" font-size="16" panose-1="2 7 3 9 2 2 5 2 4 4" units-per-em="1000" underline-position="-232.91016" underline-thickness="41.015625" slope="0" x-height="422.85156" cap-height="571.28906" ascent="832.5195" descent="-300.29297" font-weight="400">
+ <font-face-src>
+ <font-face-name name="CourierNewPSMT"/>
+ </font-face-src>
+ </font-face>
+ <filter id="Shadow" filterUnits="userSpaceOnUse" x="724" y="1659.5">
+ <feOffset in="SourceAlpha" result="offset" dx="0" dy="2"/>
+ <feFlood flood-color="#919191" flood-opacity=".25" result="flood"/>
+ <feComposite in="flood" in2="offset" operator="in" result="color"/>
+ <feMerge>
+ <feMergeNode in="color"/>
+ <feMergeNode in="SourceGraphic"/>
+ </feMerge>
+ </filter>
+ <font-face font-family="Roboto" font-size="13" panose-1="2 0 0 0 0 0 0 0 0 0" units-per-em="1000" underline-position="-73.24219" underline-thickness="48.828125" slope="0" x-height="528.3203" cap-height="710.9375" ascent="927.7344" descent="-244.14062" font-weight="700">
+ <font-face-src>
+ <font-face-name name="Roboto-Bold"/>
+ </font-face-src>
+ </font-face>
+ <font-face font-family="Courier New" font-size="13" panose-1="2 7 6 9 2 2 5 2 4 4" units-per-em="1000" underline-position="-232.91016" underline-thickness="100.09766" slope="0" x-height="443.3594" cap-height="591.7969" ascent="832.5195" descent="-300.29297" font-weight="700">
+ <font-face-src>
+ <font-face-name name="CourierNewPS-BoldMT"/>
+ </font-face-src>
+ </font-face>
+ <font-face font-family="Roboto" font-size="12" panose-1="2 0 0 0 0 0 0 0 0 0" units-per-em="1000" underline-position="-73.24219" underline-thickness="48.828125" slope="0" x-height="528.3203" cap-height="710.9375" ascent="927.7344" descent="-244.14062" font-weight="400">
+ <font-face-src>
+ <font-face-name name="Roboto-Regular"/>
+ </font-face-src>
+ </font-face>
+ <font-face font-family="Roboto" font-size="11" panose-1="2 0 0 0 0 0 0 0 0 0" units-per-em="1000" underline-position="-73.24219" underline-thickness="48.828125" slope="0" x-height="528.3203" cap-height="710.9375" ascent="927.7344" descent="-244.14062" font-weight="400">
+ <font-face-src>
+ <font-face-name name="Roboto-Regular"/>
+ </font-face-src>
+ </font-face>
+ <marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -3 5 6" markerWidth="5" markerHeight="6" color="#00aeef">
+ <g>
+ <path d="M 2.88 0 L 0 -1.08 L 0 1.08 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
+ </g>
+ </marker>
+ </defs>
+ <metadata> Produced by OmniGraffle 7.12.1
+ <dc:date>2020-02-12 20:52:25 +0000</dc:date>
+ </metadata>
+ <g id="Canvas_1" stroke="none" stroke-opacity="1" fill="none" fill-opacity="1" stroke-dasharray="none">
+ <title>Canvas 1</title>
+ <g id="Canvas_1: Layer 1">
+ <title>Layer 1</title>
+ <g id="Graphic_259">
+ <rect x="745" y="1660" width="420" height="385" fill="#4751d4" fill-opacity=".04274277"/>
+ <path d="M 745 1660 L 1165 1660 L 1165 2045 L 745 2045 Z" stroke="gray" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="1.0,4.0" stroke-width="1"/>
+ <clipPath id="clip_path">
+ <rect x="0" y="0" width="420" height="385" fill="#4751d4" fill-opacity=".04274277"/>
+ </clipPath>
+ <text clip-path="url(#clip_path)" transform="translate(750 1665)" fill="black">
+ <tspan font-family="Courier New" font-size="16" font-weight="400" fill="black" x="27.371094" y="13">Worker::run() (Multi-Threaded-Worker)</tspan>
+ </text>
+ </g>
+ <g id="Graphic_260" filter="url(#Shadow)">
+ <path d="M 825 2025 L 895 2025 C 903.28 2025 910 2033.96 910 2045 C 910 2056.04 903.28 2065 895 2065 L 825 2065 C 816.72 2065 810 2056.04 810 2045 C 810 2033.96 816.72 2025 825 2025 Z" fill="#ffc7b1"/>
+ <path d="M 825 2025 L 895 2025 C 903.28 2025 910 2033.96 910 2045 C 910 2056.04 903.28 2065 895 2065 L 825 2065 C 816.72 2065 810 2056.04 810 2045 C 810 2033.96 816.72 2025 825 2025 Z" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(816 2037.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="700" fill="#515556" x="28.946533" y="12">Done</tspan>
+ </text>
+ </g>
+ <g id="Graphic_261" filter="url(#Shadow)">
+ <path d="M 765 1851.75 L 765 1918.25 C 765 1926.116 756.04 1932.5 745 1932.5 C 733.96 1932.5 725 1926.116 725 1918.25 L 725 1851.75 C 725 1843.884 733.96 1837.5 745 1837.5 C 756.04 1837.5 765 1843.884 765 1851.75 Z" fill="#a7fee5"/>
+ <path d="M 765 1851.75 L 765 1918.25 C 765 1926.116 756.04 1932.5 745 1932.5 C 733.96 1932.5 725 1926.116 725 1918.25 L 725 1851.75 C 725 1843.884 733.96 1837.5 745 1837.5 C 756.04 1837.5 765 1843.884 765 1851.75 Z" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(752.5 1843.5) rotate(90)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="700" fill="#515556" x="27.097168" y="12">Start</tspan>
+ </text>
+ </g>
+ <g id="Graphic_262" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="1e3" y="1725" width="140" height="40" fill="#c0c0ff"/>
+ <rect x="1e3" y="1725" width="140" height="40" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1006 1737.5)" fill="#515556">
+ <tspan font-family="Courier New" font-size="13" font-weight="700" fill="#515556" x="13.291748" y="11">waitForWork()</tspan>
+ </text>
+ </g>
+ <g id="Graphic_263" filter="url(#Shadow)">
+ <path d="M 860 1855 L 925 1885 L 860 1915 L 795 1885 Z" fill="white"/>
+ <path d="M 860 1855 L 925 1885 L 860 1915 L 795 1885 Z" stroke="#fcc04d" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(801 1878)" fill="#515556">
+ <tspan font-family="Roboto" font-size="12" font-weight="400" fill="#515556" x="29.410156" y="11">Shutdown?</tspan>
+ </text>
+ </g>
+ <g id="Graphic_264" filter="url(#Shadow)">
+ <circle cx="860" cy="1960" r="15.0000239685285" fill="white"/>
+ <circle cx="860" cy="1960" r="15.0000239685285" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(851 1953.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x=".12158203" y="10">Yes</tspan>
+ </text>
+ </g>
+ <g id="Graphic_268" filter="url(#Shadow)">
+ <circle cx="860" cy="1810" r="15.0000239685285" fill="white"/>
+ <circle cx="860" cy="1810" r="15.0000239685285" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(851 1803.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x="1.9423828" y="10">No</tspan>
+ </text>
+ </g>
+ <g id="Line_269">
+ <line x1="860" y1="1853.8986" x2="860" y2="1835.06" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_270">
+ <line x1="860" y1="1916.1014" x2="860" y2="1934.94" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_271">
+ <path d="M 860 1795 L 860 1745 L 990.94 1745" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_272">
+ <line x1="765" y1="1885" x2="783.5537" y2="1885" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_275">
+ <line x1="860" y1="1975" x2="860" y2="2014.94" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Graphic_277" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="1e3" y="1800" width="140" height="40" fill="#c0c0ff"/>
+ <rect x="1e3" y="1800" width="140" height="40" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1006 1812.5)" fill="#515556">
+ <tspan font-family="Courier New" font-size="13" font-weight="700" fill="#515556" x="9.391113" y="11">runUntilIdle()</tspan>
+ </text>
+ </g>
+ <g id="Line_278">
+ <line x1="1070" y1="1765" x2="1070" y2="1789.94" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_279">
+ <path d="M 1070 1840 L 1070 1885 L 934.06 1885" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ </g>
+ </g>
+</svg>
diff --git a/docs/imgs/worker_run_stw.svg b/docs/imgs/worker_run_stw.svg
new file mode 100644
index 0000000..dbe17e2
--- /dev/null
+++ b/docs/imgs/worker_run_stw.svg
@@ -0,0 +1,161 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="264 1659.5 441.5 408.5" width="441.5" height="408.5">
+ <defs>
+ <font-face font-family="Courier New" font-size="16" panose-1="2 7 3 9 2 2 5 2 4 4" units-per-em="1000" underline-position="-232.91016" underline-thickness="41.015625" slope="0" x-height="422.85156" cap-height="571.28906" ascent="832.5195" descent="-300.29297" font-weight="400">
+ <font-face-src>
+ <font-face-name name="CourierNewPSMT"/>
+ </font-face-src>
+ </font-face>
+ <filter id="Shadow" filterUnits="userSpaceOnUse" x="264" y="1659.5">
+ <feOffset in="SourceAlpha" result="offset" dx="0" dy="2"/>
+ <feFlood flood-color="#919191" flood-opacity=".25" result="flood"/>
+ <feComposite in="flood" in2="offset" operator="in" result="color"/>
+ <feMerge>
+ <feMergeNode in="color"/>
+ <feMergeNode in="SourceGraphic"/>
+ </feMerge>
+ </filter>
+ <font-face font-family="Roboto" font-size="13" panose-1="2 0 0 0 0 0 0 0 0 0" units-per-em="1000" underline-position="-73.24219" underline-thickness="48.828125" slope="0" x-height="528.3203" cap-height="710.9375" ascent="927.7344" descent="-244.14062" font-weight="700">
+ <font-face-src>
+ <font-face-name name="Roboto-Bold"/>
+ </font-face-src>
+ </font-face>
+ <font-face font-family="Courier New" font-size="13" panose-1="2 7 6 9 2 2 5 2 4 4" units-per-em="1000" underline-position="-232.91016" underline-thickness="100.09766" slope="0" x-height="443.3594" cap-height="591.7969" ascent="832.5195" descent="-300.29297" font-weight="700">
+ <font-face-src>
+ <font-face-name name="CourierNewPS-BoldMT"/>
+ </font-face-src>
+ </font-face>
+ <font-face font-family="Roboto" font-size="12" panose-1="2 0 0 0 0 0 0 0 0 0" units-per-em="1000" underline-position="-73.24219" underline-thickness="48.828125" slope="0" x-height="528.3203" cap-height="710.9375" ascent="927.7344" descent="-244.14062" font-weight="400">
+ <font-face-src>
+ <font-face-name name="Roboto-Regular"/>
+ </font-face-src>
+ </font-face>
+ <font-face font-family="Roboto" font-size="11" panose-1="2 0 0 0 0 0 0 0 0 0" units-per-em="1000" underline-position="-73.24219" underline-thickness="48.828125" slope="0" x-height="528.3203" cap-height="710.9375" ascent="927.7344" descent="-244.14062" font-weight="400">
+ <font-face-src>
+ <font-face-name name="Roboto-Regular"/>
+ </font-face-src>
+ </font-face>
+ <font-face font-family="Roboto" font-size="13" panose-1="2 0 0 0 0 0 0 0 0 0" units-per-em="1000" underline-position="-73.24219" underline-thickness="48.828125" slope="0" x-height="528.3203" cap-height="710.9375" ascent="927.7344" descent="-244.14062" font-weight="400">
+ <font-face-src>
+ <font-face-name name="Roboto-Regular"/>
+ </font-face-src>
+ </font-face>
+ <marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -3 5 6" markerWidth="5" markerHeight="6" color="#00aeef">
+ <g>
+ <path d="M 2.88 0 L 0 -1.08 L 0 1.08 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
+ </g>
+ </marker>
+ </defs>
+ <metadata> Produced by OmniGraffle 7.12.1
+ <dc:date>2020-02-12 20:52:25 +0000</dc:date>
+ </metadata>
+ <g id="Canvas_1" stroke="none" stroke-opacity="1" fill="none" fill-opacity="1" stroke-dasharray="none">
+ <title>Canvas 1</title>
+ <g id="Canvas_1: Layer 1">
+ <title>Layer 1</title>
+ <g id="Graphic_254">
+ <rect x="285" y="1660" width="420" height="385" fill="#4751d4" fill-opacity=".04274277"/>
+ <path d="M 285 1660 L 705 1660 L 705 2045 L 285 2045 Z" stroke="gray" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="1.0,4.0" stroke-width="1"/>
+ <clipPath id="clip_path">
+ <rect x="0" y="0" width="420" height="385" fill="#4751d4" fill-opacity=".04274277"/>
+ </clipPath>
+ <text clip-path="url(#clip_path)" transform="translate(290 1665)" fill="black">
+ <tspan font-family="Courier New" font-size="16" font-weight="400" fill="black" x="22.570312" y="13">Worker::run() (Single-Threaded-Worker)</tspan>
+ </text>
+ </g>
+ <g id="Graphic_245" filter="url(#Shadow)">
+ <path d="M 365 2025 L 435 2025 C 443.28 2025 450 2033.96 450 2045 C 450 2056.04 443.28 2065 435 2065 L 365 2065 C 356.72 2065 350 2056.04 350 2045 C 350 2033.96 356.72 2025 365 2025 Z" fill="#ffc7b1"/>
+ <path d="M 365 2025 L 435 2025 C 443.28 2025 450 2033.96 450 2045 C 450 2056.04 443.28 2065 435 2065 L 365 2065 C 356.72 2065 350 2056.04 350 2045 C 350 2033.96 356.72 2025 365 2025 Z" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(356 2037.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="700" fill="#515556" x="28.946533" y="12">Done</tspan>
+ </text>
+ </g>
+ <g id="Graphic_244" filter="url(#Shadow)">
+ <path d="M 305 1851.75 L 305 1918.25 C 305 1926.116 296.04 1932.5 285 1932.5 C 273.96 1932.5 265 1926.116 265 1918.25 L 265 1851.75 C 265 1843.884 273.96 1837.5 285 1837.5 C 296.04 1837.5 305 1843.884 305 1851.75 Z" fill="#a7fee5"/>
+ <path d="M 305 1851.75 L 305 1918.25 C 305 1926.116 296.04 1932.5 285 1932.5 C 273.96 1932.5 265 1926.116 265 1918.25 L 265 1851.75 C 265 1843.884 273.96 1837.5 285 1837.5 C 296.04 1837.5 305 1843.884 305 1851.75 Z" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(292.5 1843.5) rotate(90)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="700" fill="#515556" x="27.097168" y="12">Start</tspan>
+ </text>
+ </g>
+ <g id="Graphic_243" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="540" y="1725" width="140" height="40" fill="#c0c0ff"/>
+ <rect x="540" y="1725" width="140" height="40" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(546 1737.5)" fill="#515556">
+ <tspan font-family="Courier New" font-size="13" font-weight="700" fill="#515556" x="9.391113" y="11">runUntilIdle()</tspan>
+ </text>
+ </g>
+ <g id="Graphic_242" filter="url(#Shadow)">
+ <path d="M 400 1855 L 465 1885 L 400 1915 L 335 1885 Z" fill="white"/>
+ <path d="M 400 1855 L 465 1885 L 400 1915 L 335 1885 Z" stroke="#fcc04d" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(341 1878)" fill="#515556">
+ <tspan font-family="Roboto" font-size="12" font-weight="400" fill="#515556" x="29.410156" y="11">Shutdown?</tspan>
+ </text>
+ </g>
+ <g id="Graphic_241" filter="url(#Shadow)">
+ <circle cx="400" cy="1960" r="15.0000239685285" fill="white"/>
+ <circle cx="400" cy="1960" r="15.0000239685285" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(391 1953.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x=".12158203" y="10">Yes</tspan>
+ </text>
+ </g>
+ <g id="Graphic_239" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="540" y="1805" width="140" height="40" fill="white"/>
+ <rect x="540" y="1805" width="140" height="40" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(546 1810)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="19.360107" y="12">Place Fiber into</tspan>
+ <tspan font-family="Courier New" font-size="13" font-weight="700" fill="#515556" x="24.993652" y="26">idleFibers</tspan>
+ </text>
+ </g>
+ <g id="Graphic_234" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="540" y="1880" width="140" height="30" fill="#c0ffc0"/>
+ <rect x="540" y="1880" width="140" height="30" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(546 1887.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="4.1384277" y="12">Switch To Main Fiber</tspan>
+ </text>
+ </g>
+ <g id="Graphic_233" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="540" y="1910" width="140" height="30" fill="#ffc0c0"/>
+ <rect x="540" y="1910" width="140" height="30" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(546 1917.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="40.47876" y="12">Resume</tspan>
+ </text>
+ </g>
+ <g id="Graphic_232" filter="url(#Shadow)">
+ <circle cx="400" cy="1810" r="15.0000239685285" fill="white"/>
+ <circle cx="400" cy="1810" r="15.0000239685285" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(391 1803.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x="1.9423828" y="10">No</tspan>
+ </text>
+ </g>
+ <g id="Line_223">
+ <line x1="400" y1="1853.8986" x2="400" y2="1835.06" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_222">
+ <line x1="400" y1="1916.1014" x2="400" y2="1934.94" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_219">
+ <path d="M 400 1795 L 400 1745 L 530.94 1745" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_217">
+ <line x1="305" y1="1885" x2="323.5537" y2="1885" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_255">
+ <line x1="610" y1="1766" x2="610" y2="1794.94" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_256">
+ <line x1="610" y1="1845" x2="610" y2="1870.94" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_257">
+ <line x1="400" y1="1975" x2="400" y2="2014.94" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_258">
+ <path d="M 610 1940 L 610 1980 L 500 1980 L 500 1885 L 474.06 1885" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ </g>
+ </g>
+</svg>
diff --git a/docs/imgs/worker_rununtilidle.svg b/docs/imgs/worker_rununtilidle.svg
new file mode 100644
index 0000000..de0bc6e
--- /dev/null
+++ b/docs/imgs/worker_rununtilidle.svg
@@ -0,0 +1,215 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="914 719.5 592 403.5" width="592" height="403.5">
+ <defs>
+ <font-face font-family="Courier New" font-size="16" panose-1="2 7 3 9 2 2 5 2 4 4" units-per-em="1000" underline-position="-232.91016" underline-thickness="41.015625" slope="0" x-height="422.85156" cap-height="571.28906" ascent="832.5195" descent="-300.29297" font-weight="400">
+ <font-face-src>
+ <font-face-name name="CourierNewPSMT"/>
+ </font-face-src>
+ </font-face>
+ <filter id="Shadow" filterUnits="userSpaceOnUse" x="914" y="719.5">
+ <feOffset in="SourceAlpha" result="offset" dx="0" dy="2"/>
+ <feFlood flood-color="#919191" flood-opacity=".25" result="flood"/>
+ <feComposite in="flood" in2="offset" operator="in" result="color"/>
+ <feMerge>
+ <feMergeNode in="color"/>
+ <feMergeNode in="SourceGraphic"/>
+ </feMerge>
+ </filter>
+ <font-face font-family="Roboto" font-size="13" panose-1="2 0 0 0 0 0 0 0 0 0" units-per-em="1000" underline-position="-73.24219" underline-thickness="48.828125" slope="0" x-height="528.3203" cap-height="710.9375" ascent="927.7344" descent="-244.14062" font-weight="400">
+ <font-face-src>
+ <font-face-name name="Roboto-Regular"/>
+ </font-face-src>
+ </font-face>
+ <font-face font-family="Roboto" font-size="11" panose-1="2 0 0 0 0 0 0 0 0 0" units-per-em="1000" underline-position="-73.24219" underline-thickness="48.828125" slope="0" x-height="528.3203" cap-height="710.9375" ascent="927.7344" descent="-244.14062" font-weight="400">
+ <font-face-src>
+ <font-face-name name="Roboto-Regular"/>
+ </font-face-src>
+ </font-face>
+ <marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -3 5 6" markerWidth="5" markerHeight="6" color="#00aeef">
+ <g>
+ <path d="M 2.88 0 L 0 -1.08 L 0 1.08 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
+ </g>
+ </marker>
+ <font-face font-family="Courier New" font-size="13" panose-1="2 7 6 9 2 2 5 2 4 4" units-per-em="1000" underline-position="-232.91016" underline-thickness="100.09766" slope="0" x-height="443.3594" cap-height="591.7969" ascent="832.5195" descent="-300.29297" font-weight="700">
+ <font-face-src>
+ <font-face-name name="CourierNewPS-BoldMT"/>
+ </font-face-src>
+ </font-face>
+ <font-face font-family="Roboto" font-size="13" panose-1="2 0 0 0 0 0 0 0 0 0" units-per-em="1000" underline-position="-73.24219" underline-thickness="48.828125" slope="0" x-height="528.3203" cap-height="710.9375" ascent="927.7344" descent="-244.14062" font-weight="700">
+ <font-face-src>
+ <font-face-name name="Roboto-Bold"/>
+ </font-face-src>
+ </font-face>
+ </defs>
+ <metadata> Produced by OmniGraffle 7.12.1
+ <dc:date>2020-02-12 20:48:56 +0000</dc:date>
+ </metadata>
+ <g id="Canvas_1" stroke="none" stroke-opacity="1" fill="none" fill-opacity="1" stroke-dasharray="none">
+ <title>Canvas 1</title>
+ <g id="Canvas_1: Layer 1">
+ <title>Layer 1</title>
+ <g id="Graphic_86">
+ <rect x="935" y="720" width="550" height="402.5" fill="#4751d4" fill-opacity=".04274277"/>
+ <path d="M 935 720 L 1485 720 L 1485 1122.5 L 935 1122.5 Z" stroke="gray" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="1.0,4.0" stroke-width="1"/>
+ <clipPath id="clip_path">
+ <rect x="0" y="0" width="550" height="402.5" fill="#4751d4" fill-opacity=".04274277"/>
+ </clipPath>
+ <text clip-path="url(#clip_path)" transform="translate(940 725)" fill="black">
+ <tspan font-family="Courier New" font-size="16" font-weight="400" fill="black" x="164.38281" y="13">Worker::runUntilIdle()</tspan>
+ </text>
+ </g>
+ <g id="Graphic_87" filter="url(#Shadow)">
+ <path d="M 1110 762.5 L 1175 792.5 L 1110 822.5 L 1045 792.5 Z" fill="white"/>
+ <path d="M 1110 762.5 L 1175 792.5 L 1110 822.5 L 1045 792.5 Z" stroke="#fcc04d" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1051 777.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="24.906738" y="12">Have Fibers</tspan>
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="38.20825" y="27">Ready?</tspan>
+ </text>
+ </g>
+ <g id="Graphic_89" filter="url(#Shadow)">
+ <circle cx="1110" cy="857.5" r="15.0000239685285" fill="white"/>
+ <circle cx="1110" cy="857.5" r="15.0000239685285" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1101 851)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x=".12158203" y="10">Yes</tspan>
+ </text>
+ </g>
+ <g id="Graphic_90" filter="url(#Shadow)">
+ <circle cx="1215" cy="792.5" r="15.0000239685284" fill="white"/>
+ <circle cx="1215" cy="792.5" r="15.0000239685284" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1206 786)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x="1.9423828" y="10">No</tspan>
+ </text>
+ </g>
+ <g id="Line_94">
+ <line x1="1177.3863" y1="792.5" x2="1189.94" y2="792.5" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_99">
+ <line x1="1110" y1="823.6014" x2="1110" y2="832.44" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Graphic_108" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="1045" y="887.5" width="130" height="40" fill="white"/>
+ <rect x="1045" y="887.5" width="130" height="40" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1051 892.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="12.744629" y="12">Take Fiber From</tspan>
+ <tspan font-family="Courier New" font-size="13" font-weight="700" fill="#515556" x="16.093018" y="26">work.fibers</tspan>
+ </text>
+ </g>
+ <g id="Graphic_109" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="1045" y="947.5" width="130" height="40" fill="white"/>
+ <rect x="1045" y="947.5" width="130" height="40" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1051 952.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="9.161377" y="12">Add This Fiber To </tspan>
+ <tspan font-family="Courier New" font-size="13" font-weight="700" fill="#515556" x="19.993652" y="26">idleFibers</tspan>
+ </text>
+ </g>
+ <g id="Graphic_110" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="1045" y="1007.5" width="130" height="30" fill="#c0ffc0"/>
+ <rect x="1045" y="1007.5" width="130" height="30" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1051 1015)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="15.121826" y="12">Switch To Fiber</tspan>
+ </text>
+ </g>
+ <g id="Graphic_111" filter="url(#Shadow)">
+ <path d="M 1320 762.5 L 1385 792.5 L 1320 822.5 L 1255 792.5 Z" fill="white"/>
+ <path d="M 1320 762.5 L 1385 792.5 L 1320 822.5 L 1255 792.5 Z" stroke="#fcc04d" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1261 777.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="26.008057" y="12">Have Tasks</tspan>
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="38.20825" y="27">Ready?</tspan>
+ </text>
+ </g>
+ <g id="Line_112">
+ <line x1="1231" y1="792.5" x2="1243.5537" y2="792.5" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Graphic_113" filter="url(#Shadow)">
+ <circle cx="1425" cy="792.5" r="15.0000239685285" fill="white"/>
+ <circle cx="1425" cy="792.5" r="15.0000239685285" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1416 786)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x="1.9423828" y="10">No</tspan>
+ </text>
+ </g>
+ <g id="Line_114">
+ <line x1="1387.3863" y1="792.5" x2="1399.94" y2="792.5" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Graphic_115" filter="url(#Shadow)">
+ <circle cx="1320" cy="857.5" r="15.0000239685285" fill="white"/>
+ <circle cx="1320" cy="857.5" r="15.0000239685285" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1311 851)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x=".12158203" y="10">Yes</tspan>
+ </text>
+ </g>
+ <g id="Line_116">
+ <line x1="1320" y1="823.6014" x2="1320" y2="832.44" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Graphic_117" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="1255" y="912.5" width="130" height="40" fill="white"/>
+ <rect x="1255" y="912.5" width="130" height="40" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1261 917.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="13.845947" y="12">Take Task From</tspan>
+ <tspan font-family="Courier New" font-size="13" font-weight="700" fill="#515556" x="19.993652" y="26">work.tasks</tspan>
+ </text>
+ </g>
+ <g id="Graphic_118" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="1255" y="987.5" width="130" height="40" fill="white"/>
+ <rect x="1255" y="987.5" width="130" height="40" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1261 1e3)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="33.111084" y="12">Call Task</tspan>
+ </text>
+ </g>
+ <g id="Line_119">
+ <path d="M 1110 873.5 L 1110 880 L 1110 877.97 L 1110 878.44" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_120">
+ <line x1="1110" y1="928.5" x2="1110" y2="937.44" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_121">
+ <line x1="1110" y1="988.5" x2="1110" y2="997.44" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Graphic_122" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="1045" y="1037.5" width="130" height="30" fill="#ffc0c0"/>
+ <rect x="1045" y="1037.5" width="130" height="30" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1051 1045)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="35.47876" y="12">Resume</tspan>
+ </text>
+ </g>
+ <g id="Line_124">
+ <line x1="1320" y1="953.5" x2="1320" y2="977.44" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_125">
+ <line x1="1320" y1="873.5" x2="1320" y2="902.44" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_126">
+ <path d="M 1110 1067.5 L 1110 1092.5 L 1005 1092.5 L 1005.1758 837.53125 L 1005.1758 792.5 L 1033.5537 792.5" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_129">
+ <path d="M 1320 1027.5 L 1320 1092.5 L 1005 1092.5" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Graphic_130" filter="url(#Shadow)">
+ <path d="M 1505 899.25 L 1505 965.75 C 1505 973.616 1496.04 980 1485 980 C 1473.96 980 1465 973.616 1465 965.75 L 1465 899.25 C 1465 891.384 1473.96 885 1485 885 C 1496.04 885 1505 891.384 1505 899.25 Z" fill="#ffc7b1"/>
+ <path d="M 1505 899.25 L 1505 965.75 C 1505 973.616 1496.04 980 1485 980 C 1473.96 980 1465 973.616 1465 965.75 L 1465 899.25 C 1465 891.384 1473.96 885 1485 885 C 1496.04 885 1505 891.384 1505 899.25 Z" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1492.5 891) rotate(90)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="700" fill="#515556" x="26.446533" y="12">Done</tspan>
+ </text>
+ </g>
+ <g id="Line_131">
+ <path d="M 1425 808.5 L 1425 932.5 L 1454.94 932.5" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Graphic_132" filter="url(#Shadow)">
+ <path d="M 955 899.25 L 955 965.75 C 955 973.616 946.04 980 935 980 C 923.96 980 915 973.616 915 965.75 L 915 899.25 C 915 891.384 923.96 885 935 885 C 946.04 885 955 891.384 955 899.25 Z" fill="#a7fee5"/>
+ <path d="M 955 899.25 L 955 965.75 C 955 973.616 946.04 980 935 980 C 923.96 980 915 973.616 915 965.75 L 915 899.25 C 915 891.384 923.96 885 935 885 C 946.04 885 955 891.384 955 899.25 Z" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(942.5 891) rotate(90)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="700" fill="#515556" x="27.097168" y="12">Start</tspan>
+ </text>
+ </g>
+ <g id="Line_134">
+ <path d="M 956 932.5 L 966 932.5 L 980.0551 932.6235 L 990.0501 932.747 L 996.0501 932.747" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ </g>
+ </g>
+</svg>
diff --git a/docs/imgs/worker_spinforwork.svg b/docs/imgs/worker_spinforwork.svg
new file mode 100644
index 0000000..83857ef
--- /dev/null
+++ b/docs/imgs/worker_spinforwork.svg
@@ -0,0 +1,195 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="169 629.5 407 481" width="407" height="481">
+ <defs>
+ <font-face font-family="Courier New" font-size="16" panose-1="2 7 3 9 2 2 5 2 4 4" units-per-em="1000" underline-position="-232.91016" underline-thickness="41.015625" slope="0" x-height="422.85156" cap-height="571.28906" ascent="832.5195" descent="-300.29297" font-weight="400">
+ <font-face-src>
+ <font-face-name name="CourierNewPSMT"/>
+ </font-face-src>
+ </font-face>
+ <filter id="Shadow" filterUnits="userSpaceOnUse" x="169" y="629.5">
+ <feOffset in="SourceAlpha" result="offset" dx="0" dy="2"/>
+ <feFlood flood-color="#919191" flood-opacity=".25" result="flood"/>
+ <feComposite in="flood" in2="offset" operator="in" result="color"/>
+ <feMerge>
+ <feMergeNode in="color"/>
+ <feMergeNode in="SourceGraphic"/>
+ </feMerge>
+ </filter>
+ <font-face font-family="Roboto" font-size="13" panose-1="2 0 0 0 0 0 0 0 0 0" units-per-em="1000" underline-position="-73.24219" underline-thickness="48.828125" slope="0" x-height="528.3203" cap-height="710.9375" ascent="927.7344" descent="-244.14062" font-weight="400">
+ <font-face-src>
+ <font-face-name name="Roboto-Regular"/>
+ </font-face-src>
+ </font-face>
+ <font-face font-family="Courier New" font-size="13" panose-1="2 7 6 9 2 2 5 2 4 4" units-per-em="1000" underline-position="-232.91016" underline-thickness="100.09766" slope="0" x-height="443.3594" cap-height="591.7969" ascent="832.5195" descent="-300.29297" font-weight="700">
+ <font-face-src>
+ <font-face-name name="CourierNewPS-BoldMT"/>
+ </font-face-src>
+ </font-face>
+ <font-face font-family="Roboto" font-size="11" panose-1="2 0 0 0 0 0 0 0 0 0" units-per-em="1000" underline-position="-73.24219" underline-thickness="48.828125" slope="0" x-height="528.3203" cap-height="710.9375" ascent="927.7344" descent="-244.14062" font-weight="400">
+ <font-face-src>
+ <font-face-name name="Roboto-Regular"/>
+ </font-face-src>
+ </font-face>
+ <marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -3 5 6" markerWidth="5" markerHeight="6" color="#00aeef">
+ <g>
+ <path d="M 2.88 0 L 0 -1.08 L 0 1.08 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
+ </g>
+ </marker>
+ <font-face font-family="Roboto" font-size="13" panose-1="2 0 0 0 0 0 0 0 0 0" units-per-em="1000" underline-position="-73.24219" underline-thickness="48.828125" slope="0" x-height="528.3203" cap-height="710.9375" ascent="927.7344" descent="-244.14062" font-weight="700">
+ <font-face-src>
+ <font-face-name name="Roboto-Bold"/>
+ </font-face-src>
+ </font-face>
+ </defs>
+ <metadata> Produced by OmniGraffle 7.12.1
+ <dc:date>2020-02-12 20:47:06 +0000</dc:date>
+ </metadata>
+ <g id="Canvas_1" stroke="none" stroke-opacity="1" fill="none" fill-opacity="1" stroke-dasharray="none">
+ <title>Canvas 1</title>
+ <g id="Canvas_1: Layer 1">
+ <title>Layer 1</title>
+ <g id="Graphic_17">
+ <rect x="190" y="630" width="365" height="480" fill="#4751d4" fill-opacity=".04274277"/>
+ <path d="M 190 630 L 555 630 L 555 1110 L 190 1110 Z" stroke="gray" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="1.0,4.0" stroke-width="1"/>
+ <clipPath id="clip_path">
+ <rect x="0" y="0" width="365" height="480" fill="#4751d4" fill-opacity=".04274277"/>
+ </clipPath>
+ <text clip-path="url(#clip_path)" transform="translate(195 635)" fill="black">
+ <tspan font-family="Courier New" font-size="16" font-weight="400" fill="black" x="76.68359" y="13">Worker::spinForWork()</tspan>
+ </text>
+ </g>
+ <g id="Graphic_18" filter="url(#Shadow)">
+ <path d="M 355 790 L 420 815 L 355 840 L 290 815 Z" fill="white"/>
+ <path d="M 355 790 L 420 815 L 355 840 L 290 815 Z" stroke="#fcc04d" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(296 807.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="24.779785" y="12">Have Work?</tspan>
+ </text>
+ </g>
+ <g id="Graphic_21" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="290" y="720" width="130" height="35.430693" fill="white"/>
+ <rect x="290" y="720" width="130" height="35.430693" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(296 729.71535)" fill="#515556">
+ <tspan font-family="Courier New" font-size="13" font-weight="700" fill="#515556" x="36.22461" y="12">nop</tspan>
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" y="12"> x N</tspan>
+ </text>
+ </g>
+ <g id="Graphic_22" filter="url(#Shadow)">
+ <circle cx="455" cy="815" r="15.0000239685285" fill="white"/>
+ <circle cx="455" cy="815" r="15.0000239685285" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(446 808.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x=".12158203" y="10">Yes</tspan>
+ </text>
+ </g>
+ <g id="Graphic_23" filter="url(#Shadow)">
+ <circle cx="355" cy="880" r="15.0000239685284" fill="white"/>
+ <circle cx="355" cy="880" r="15.0000239685284" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(346 873.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x="1.9423828" y="10">No</tspan>
+ </text>
+ </g>
+ <g id="Graphic_24" filter="url(#Shadow)">
+ <path d="M 355 1045 L 420 1070 L 355 1095 L 290 1070 Z" fill="white"/>
+ <path d="M 355 1045 L 420 1070 L 355 1095 L 290 1070 Z" stroke="#fcc04d" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(296 1062.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="17.829102" y="12">Spin Timeout?</tspan>
+ </text>
+ </g>
+ <g id="Graphic_29" filter="url(#Shadow)">
+ <circle cx="255" cy="1070" r="15.0000239685285" fill="white"/>
+ <circle cx="255" cy="1070" r="15.0000239685285" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(246 1063.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x="1.9423828" y="10">No</tspan>
+ </text>
+ </g>
+ <g id="Line_35">
+ <line x1="355" y1="755.4307" x2="355" y2="779.8686" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_36">
+ <line x1="355" y1="841.0714" x2="355" y2="854.94" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Graphic_40" filter="url(#Shadow)">
+ <circle cx="455" cy="1070" r="15.0000239685285" fill="white"/>
+ <circle cx="455" cy="1070" r="15.0000239685285" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(446 1063.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x=".12158203" y="10">Yes</tspan>
+ </text>
+ </g>
+ <g id="Line_41">
+ <path d="M 255 1054 L 255 690 L 355 690 L 355 710.94" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_42">
+ <line x1="339" y1="880" x2="264.06" y2="880" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-dasharray="2.0,8.0" stroke-width="2"/>
+ </g>
+ <g id="Line_43">
+ <line x1="287.21432" y1="1070" x2="280.06" y2="1070" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_44">
+ <line x1="422.7857" y1="815" x2="429.94" y2="815" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_45">
+ <line x1="422.7857" y1="1070" x2="429.94" y2="1070" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Graphic_50" filter="url(#Shadow)">
+ <path d="M 355 920 L 420 945 L 355 970 L 290 945 Z" fill="white"/>
+ <path d="M 355 920 L 420 945 L 355 970 L 290 945 Z" stroke="#fcc04d" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(296 937.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="24.91626" y="12">Steal Work?</tspan>
+ </text>
+ </g>
+ <g id="Line_51">
+ <line x1="355" y1="896" x2="355" y2="909.8686" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Graphic_52" filter="url(#Shadow)">
+ <circle cx="355" cy="1007.5" r="15.0000239685284" fill="white"/>
+ <circle cx="355" cy="1007.5" r="15.0000239685284" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(346 1001)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x="1.9423828" y="10">No</tspan>
+ </text>
+ </g>
+ <g id="Line_53">
+ <line x1="355" y1="971.0714" x2="355" y2="982.44" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_54">
+ <line x1="355" y1="1023.5" x2="355" y2="1034.8686" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Graphic_55" filter="url(#Shadow)">
+ <circle cx="455" cy="945" r="15.0000239685285" fill="white"/>
+ <circle cx="455" cy="945" r="15.0000239685285" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(446 938.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x=".12158203" y="10">Yes</tspan>
+ </text>
+ </g>
+ <g id="Line_56">
+ <line x1="422.7857" y1="945" x2="429.94" y2="945" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Graphic_211" filter="url(#Shadow)">
+ <path d="M 210 824.25 L 210 890.75 C 210 898.616 201.04 905 190 905 C 178.96 905 170 898.616 170 890.75 L 170 824.25 C 170 816.384 178.96 810 190 810 C 201.04 810 210 816.384 210 824.25 Z" fill="#a7fee5"/>
+ <path d="M 210 824.25 L 210 890.75 C 210 898.616 201.04 905 190 905 C 178.96 905 170 898.616 170 890.75 L 170 824.25 C 170 816.384 178.96 810 190 810 C 201.04 810 210 816.384 210 824.25 Z" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(197.5 816) rotate(90)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="700" fill="#515556" x="27.097168" y="12">Start</tspan>
+ </text>
+ </g>
+ <g id="Graphic_212" filter="url(#Shadow)">
+ <path d="M 575 824.25 L 575 890.75 C 575 898.616 566.04 905 555 905 C 543.96 905 535 898.616 535 890.75 L 535 824.25 C 535 816.384 543.96 810 555 810 C 566.04 810 575 816.384 575 824.25 Z" fill="#ffc7b1"/>
+ <path d="M 575 824.25 L 575 890.75 C 575 898.616 566.04 905 555 905 C 543.96 905 535 898.616 535 890.75 L 535 824.25 C 535 816.384 543.96 810 555 810 C 566.04 810 575 816.384 575 824.25 Z" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(562.5 816) rotate(90)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="700" fill="#515556" x="26.446533" y="12">Done</tspan>
+ </text>
+ </g>
+ <g id="Line_213">
+ <path d="M 210 857.5 L 255 857.5 L 255 785.668" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_214">
+ <path d="M 470 815 L 502.5 815 L 502.5 857.5 L 525.94 857.5" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_215">
+ <path d="M 470 945 L 502.5 945 L 502.5 857.5 L 525.94 857.5" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_216">
+ <path d="M 470 1070 L 502.5 1070 L 502.5 857.5 L 525.94 857.5" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ </g>
+ </g>
+</svg>
diff --git a/docs/imgs/worker_suspend.svg b/docs/imgs/worker_suspend.svg
new file mode 100644
index 0000000..4756c4a
--- /dev/null
+++ b/docs/imgs/worker_suspend.svg
@@ -0,0 +1,266 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="829 1174.5 811.5 463.5" width="811.5" height="463.5">
+ <defs>
+ <font-face font-family="Courier New" font-size="16" panose-1="2 7 3 9 2 2 5 2 4 4" units-per-em="1000" underline-position="-232.91016" underline-thickness="41.015625" slope="0" x-height="422.85156" cap-height="571.28906" ascent="832.5195" descent="-300.29297" font-weight="400">
+ <font-face-src>
+ <font-face-name name="CourierNewPSMT"/>
+ </font-face-src>
+ </font-face>
+ <filter id="Shadow" filterUnits="userSpaceOnUse" x="829" y="1174.5">
+ <feOffset in="SourceAlpha" result="offset" dx="0" dy="2"/>
+ <feFlood flood-color="#919191" flood-opacity=".25" result="flood"/>
+ <feComposite in="flood" in2="offset" operator="in" result="color"/>
+ <feMerge>
+ <feMergeNode in="color"/>
+ <feMergeNode in="SourceGraphic"/>
+ </feMerge>
+ </filter>
+ <font-face font-family="Roboto" font-size="12" panose-1="2 0 0 0 0 0 0 0 0 0" units-per-em="1000" underline-position="-73.24219" underline-thickness="48.828125" slope="0" x-height="528.3203" cap-height="710.9375" ascent="927.7344" descent="-244.14062" font-weight="400">
+ <font-face-src>
+ <font-face-name name="Roboto-Regular"/>
+ </font-face-src>
+ </font-face>
+ <font-face font-family="Roboto" font-size="11" panose-1="2 0 0 0 0 0 0 0 0 0" units-per-em="1000" underline-position="-73.24219" underline-thickness="48.828125" slope="0" x-height="528.3203" cap-height="710.9375" ascent="927.7344" descent="-244.14062" font-weight="400">
+ <font-face-src>
+ <font-face-name name="Roboto-Regular"/>
+ </font-face-src>
+ </font-face>
+ <marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -3 5 6" markerWidth="5" markerHeight="6" color="#00aeef">
+ <g>
+ <path d="M 2.88 0 L 0 -1.08 L 0 1.08 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
+ </g>
+ </marker>
+ <font-face font-family="Roboto" font-size="13" panose-1="2 0 0 0 0 0 0 0 0 0" units-per-em="1000" underline-position="-73.24219" underline-thickness="48.828125" slope="0" x-height="528.3203" cap-height="710.9375" ascent="927.7344" descent="-244.14062" font-weight="700">
+ <font-face-src>
+ <font-face-name name="Roboto-Bold"/>
+ </font-face-src>
+ </font-face>
+ <font-face font-family="Courier New" font-size="13" panose-1="2 7 6 9 2 2 5 2 4 4" units-per-em="1000" underline-position="-232.91016" underline-thickness="100.09766" slope="0" x-height="443.3594" cap-height="591.7969" ascent="832.5195" descent="-300.29297" font-weight="700">
+ <font-face-src>
+ <font-face-name name="CourierNewPS-BoldMT"/>
+ </font-face-src>
+ </font-face>
+ <font-face font-family="Roboto" font-size="13" panose-1="2 0 0 0 0 0 0 0 0 0" units-per-em="1000" underline-position="-73.24219" underline-thickness="48.828125" slope="0" x-height="528.3203" cap-height="710.9375" ascent="927.7344" descent="-244.14062" font-weight="400">
+ <font-face-src>
+ <font-face-name name="Roboto-Regular"/>
+ </font-face-src>
+ </font-face>
+ </defs>
+ <metadata> Produced by OmniGraffle 7.12.1
+ <dc:date>2020-02-12 20:51:35 +0000</dc:date>
+ </metadata>
+ <g id="Canvas_1" stroke="none" stroke-opacity="1" fill="none" fill-opacity="1" stroke-dasharray="none">
+ <title>Canvas 1</title>
+ <g id="Canvas_1: Layer 1">
+ <title>Layer 1</title>
+ <g id="Graphic_135">
+ <rect x="850" y="1175" width="790" height="440" fill="#4751d4" fill-opacity=".04274277"/>
+ <path d="M 850 1175 L 1640 1175 L 1640 1615 L 850 1615 Z" stroke="gray" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="1.0,4.0" stroke-width="1"/>
+ <clipPath id="clip_path">
+ <rect x="0" y="0" width="790" height="440" fill="#4751d4" fill-opacity=".04274277"/>
+ </clipPath>
+ <text clip-path="url(#clip_path)" transform="translate(855 1180)" fill="black">
+ <tspan font-family="Courier New" font-size="16" font-weight="400" fill="black" x="308.38672" y="13">Worker::suspend()</tspan>
+ </text>
+ </g>
+ <g id="Graphic_136" filter="url(#Shadow)">
+ <path d="M 1140 1305 L 1205 1335 L 1140 1365 L 1075 1335 Z" fill="white"/>
+ <path d="M 1140 1305 L 1205 1335 L 1140 1365 L 1075 1335 Z" stroke="#fcc04d" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1081 1321)" fill="#515556">
+ <tspan font-family="Roboto" font-size="12" font-weight="400" fill="#515556" x="27.529297" y="11">Have Fibers</tspan>
+ <tspan font-family="Roboto" font-size="12" font-weight="400" fill="#515556" x="39.807617" y="25">Ready?</tspan>
+ </text>
+ </g>
+ <g id="Graphic_137" filter="url(#Shadow)">
+ <circle cx="1245" cy="1335" r="15.0000239685284" fill="white"/>
+ <circle cx="1245" cy="1335" r="15.0000239685284" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1236 1328.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x=".12158203" y="10">Yes</tspan>
+ </text>
+ </g>
+ <g id="Graphic_138" filter="url(#Shadow)">
+ <circle cx="1140" cy="1400" r="15.0000239685285" fill="white"/>
+ <circle cx="1140" cy="1400" r="15.0000239685285" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1131 1393.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x="1.9423828" y="10">No</tspan>
+ </text>
+ </g>
+ <g id="Line_139">
+ <line x1="1140" y1="1366.1014" x2="1140" y2="1374.94" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_140">
+ <line x1="1207.3863" y1="1335" x2="1219.94" y2="1335" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Graphic_144" filter="url(#Shadow)">
+ <path d="M 1140 1435 L 1205 1465 L 1140 1495 L 1075 1465 Z" fill="white"/>
+ <path d="M 1140 1435 L 1205 1465 L 1140 1495 L 1075 1465 Z" stroke="#fcc04d" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1081 1458)" fill="#515556">
+ <tspan font-family="Roboto" font-size="12" font-weight="400" fill="#515556" x="16.654297" y="11">Have Idle Fiber?</tspan>
+ </text>
+ </g>
+ <g id="Line_145">
+ <line x1="1140" y1="1416" x2="1140" y2="1424.8386" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Graphic_148" filter="url(#Shadow)">
+ <circle cx="1245" cy="1465" r="15.0000239685284" fill="white"/>
+ <circle cx="1245" cy="1465" r="15.0000239685284" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1236 1458.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x=".12158203" y="10">Yes</tspan>
+ </text>
+ </g>
+ <g id="Graphic_160" filter="url(#Shadow)">
+ <path d="M 1510 1595 L 1580 1595 C 1588.28 1595 1595 1603.96 1595 1615 C 1595 1626.04 1588.28 1635 1580 1635 L 1510 1635 C 1501.72 1635 1495 1626.04 1495 1615 C 1495 1603.96 1501.72 1595 1510 1595 Z" fill="#ffc7b1"/>
+ <path d="M 1510 1595 L 1580 1595 C 1588.28 1595 1595 1603.96 1595 1615 C 1595 1626.04 1588.28 1635 1580 1635 L 1510 1635 C 1501.72 1635 1495 1626.04 1495 1615 C 1495 1603.96 1501.72 1595 1510 1595 Z" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1501 1607.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="700" fill="#515556" x="28.946533" y="12">Done</tspan>
+ </text>
+ </g>
+ <g id="Graphic_162" filter="url(#Shadow)">
+ <path d="M 870 1366.75 L 870 1433.25 C 870 1441.116 861.04 1447.5 850 1447.5 C 838.96 1447.5 830 1441.116 830 1433.25 L 830 1366.75 C 830 1358.884 838.96 1352.5 850 1352.5 C 861.04 1352.5 870 1358.884 870 1366.75 Z" fill="#a7fee5"/>
+ <path d="M 870 1366.75 L 870 1433.25 C 870 1441.116 861.04 1447.5 850 1447.5 C 838.96 1447.5 830 1441.116 830 1433.25 L 830 1366.75 C 830 1358.884 838.96 1352.5 850 1352.5 C 861.04 1352.5 870 1358.884 870 1366.75 Z" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(857.5 1358.5) rotate(90)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="700" fill="#515556" x="27.097168" y="12">Start</tspan>
+ </text>
+ </g>
+ <g id="Graphic_164" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="1075" y="1240" width="130" height="40" fill="#c0c0ff"/>
+ <rect x="1075" y="1240" width="130" height="40" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1081 1252.5)" fill="#515556">
+ <tspan font-family="Courier New" font-size="13" font-weight="700" fill="#515556" x="8.291748" y="11">waitForWork()</tspan>
+ </text>
+ </g>
+ <g id="Graphic_166" filter="url(#Shadow)">
+ <path d="M 965 1370 L 1030 1400 L 965 1430 L 900 1400 Z" fill="white"/>
+ <path d="M 965 1370 L 1030 1400 L 965 1430 L 900 1400 Z" stroke="#fcc04d" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(906 1386)" fill="#515556">
+ <tspan font-family="Roboto" font-size="12" font-weight="400" fill="#515556" x="22.270508" y="11">Suspend With</tspan>
+ <tspan font-family="Roboto" font-size="12" font-weight="400" fill="#515556" x="34.003906" y="25">Timeout?</tspan>
+ </text>
+ </g>
+ <g id="Graphic_167" filter="url(#Shadow)">
+ <circle cx="965" cy="1470" r="15.0000239685285" fill="white"/>
+ <circle cx="965" cy="1470" r="15.0000239685285" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(956 1463.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x=".12158203" y="10">Yes</tspan>
+ </text>
+ </g>
+ <g id="Graphic_168" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="900" y="1515" width="130" height="40" fill="white"/>
+ <rect x="900" y="1515" width="130" height="40" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(906 1520)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="24.01172" y="12">Add Fiber to</tspan>
+ <tspan font-family="Courier New" font-size="13" font-weight="700" fill="#515556" x="12.192383" y="26">work.waiting</tspan>
+ </text>
+ </g>
+ <g id="Graphic_170" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="1277.5" y="1315" width="130" height="40" fill="white"/>
+ <rect x="1277.5" y="1315" width="130" height="40" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1283.5 1320)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="15.569336" y="12">Take fiber from</tspan>
+ <tspan font-family="Courier New" font-size="13" font-weight="700" fill="#515556" x="16.093018" y="26">work.fibers</tspan>
+ </text>
+ </g>
+ <g id="Graphic_171" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="1275" y="1445" width="130" height="40" fill="white"/>
+ <rect x="1275" y="1445" width="130" height="40" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1281 1450)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="15.569336" y="12">Take fiber from</tspan>
+ <tspan font-family="Courier New" font-size="13" font-weight="700" fill="#515556" x="19.993652" y="26">idleFibers</tspan>
+ </text>
+ </g>
+ <g id="Line_172">
+ <line x1="1207.3863" y1="1465" x2="1219.94" y2="1465" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_173">
+ <line x1="1261" y1="1465" x2="1264.94" y2="1465" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_174">
+ <line x1="1261" y1="1335" x2="1267.44" y2="1335" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Graphic_175" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="1480" y="1470" width="130" height="30" fill="#c0ffc0"/>
+ <rect x="1480" y="1470" width="130" height="30" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1486 1477.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="15.121826" y="12">Switch To Fiber</tspan>
+ </text>
+ </g>
+ <g id="Graphic_176" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="1480" y="1500" width="130" height="30" fill="#ffc0c0"/>
+ <rect x="1480" y="1500" width="130" height="30" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1486 1507.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="35.47876" y="12">Resume</tspan>
+ </text>
+ </g>
+ <g id="Graphic_178" filter="url(#Shadow)">
+ <circle cx="965" cy="1330" r="15.0000239685285" fill="white"/>
+ <circle cx="965" cy="1330" r="15.0000239685285" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(956 1323.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x="1.9423828" y="10">No</tspan>
+ </text>
+ </g>
+ <g id="Graphic_182" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="1275" y="1545" width="130" height="40" fill="white"/>
+ <rect x="1275" y="1545" width="130" height="40" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1281 1549.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="9.443848" y="12">Create New Fiber</tspan>
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="18.397217" y="27">To Call </tspan>
+ <tspan font-family="Courier New" font-size="13" font-weight="700" fill="#515556" y="27">run()</tspan>
+ </text>
+ </g>
+ <g id="Line_187">
+ <path d="M 1140 1545 L 1140 1565 L 1264.94 1565" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Graphic_188" filter="url(#Shadow)">
+ <circle cx="1140" cy="1530" r="15.0000239685285" fill="white"/>
+ <circle cx="1140" cy="1530" r="15.0000239685285" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(1131 1523.5)" fill="#515556">
+ <tspan font-family="Roboto" font-size="11" font-weight="400" fill="#515556" x="1.9423828" y="10">No</tspan>
+ </text>
+ </g>
+ <g id="Line_189">
+ <line x1="1140" y1="1496.1014" x2="1140" y2="1504.94" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_190">
+ <path d="M 1407.5 1335 L 1430 1335 L 1430 1400 L 1545 1400 L 1545 1460.94" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_191">
+ <path d="M 1405 1465 L 1430 1465 L 1430 1400 L 1545 1400 L 1545 1460.94" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_192">
+ <path d="M 1405 1565 L 1430 1565 L 1430 1400 L 1545 1400 L 1545 1460.94" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_195">
+ <line x1="1545" y1="1530" x2="1545" y2="1584.94" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_196">
+ <line x1="965" y1="1368.8986" x2="965" y2="1355.06" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_197">
+ <line x1="965" y1="1431.1014" x2="965" y2="1444.94" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_198">
+ <line x1="965" y1="1485" x2="965" y2="1504.94" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_199">
+ <path d="M 1030 1535 L 1052.5 1535 L 1052.5 1260 L 1065.94 1260" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_200">
+ <path d="M 965 1315 L 965 1260 L 1065.94 1260" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_201">
+ <line x1="1140" y1="1280" x2="1140" y2="1295.94" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_202">
+ <line x1="870" y1="1400" x2="888.5537" y2="1400" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ </g>
+ </g>
+</svg>
diff --git a/docs/imgs/worker_waitforwork.svg b/docs/imgs/worker_waitforwork.svg
new file mode 100644
index 0000000..9c0eecc
--- /dev/null
+++ b/docs/imgs/worker_waitforwork.svg
@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="304 1234.5 377 343.5" width="377" height="343.5">
+ <defs>
+ <font-face font-family="Courier New" font-size="16" panose-1="2 7 3 9 2 2 5 2 4 4" units-per-em="1000" underline-position="-232.91016" underline-thickness="41.015625" slope="0" x-height="422.85156" cap-height="571.28906" ascent="832.5195" descent="-300.29297" font-weight="400">
+ <font-face-src>
+ <font-face-name name="CourierNewPSMT"/>
+ </font-face-src>
+ </font-face>
+ <filter id="Shadow" filterUnits="userSpaceOnUse" x="304" y="1234.5">
+ <feOffset in="SourceAlpha" result="offset" dx="0" dy="2"/>
+ <feFlood flood-color="#919191" flood-opacity=".25" result="flood"/>
+ <feComposite in="flood" in2="offset" operator="in" result="color"/>
+ <feMerge>
+ <feMergeNode in="color"/>
+ <feMergeNode in="SourceGraphic"/>
+ </feMerge>
+ </filter>
+ <font-face font-family="Roboto" font-size="13" panose-1="2 0 0 0 0 0 0 0 0 0" units-per-em="1000" underline-position="-73.24219" underline-thickness="48.828125" slope="0" x-height="528.3203" cap-height="710.9375" ascent="927.7344" descent="-244.14062" font-weight="400">
+ <font-face-src>
+ <font-face-name name="Roboto-Regular"/>
+ </font-face-src>
+ </font-face>
+ <font-face font-family="Courier New" font-size="13" panose-1="2 7 6 9 2 2 5 2 4 4" units-per-em="1000" underline-position="-232.91016" underline-thickness="100.09766" slope="0" x-height="443.3594" cap-height="591.7969" ascent="832.5195" descent="-300.29297" font-weight="700">
+ <font-face-src>
+ <font-face-name name="CourierNewPS-BoldMT"/>
+ </font-face-src>
+ </font-face>
+ <font-face font-family="Roboto" font-size="13" panose-1="2 0 0 0 0 0 0 0 0 0" units-per-em="1000" underline-position="-73.24219" underline-thickness="48.828125" slope="0" x-height="528.3203" cap-height="710.9375" ascent="927.7344" descent="-244.14062" font-weight="700">
+ <font-face-src>
+ <font-face-name name="Roboto-Bold"/>
+ </font-face-src>
+ </font-face>
+ <marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -3 5 6" markerWidth="5" markerHeight="6" color="#00aeef">
+ <g>
+ <path d="M 2.88 0 L 0 -1.08 L 0 1.08 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
+ </g>
+ </marker>
+ </defs>
+ <metadata> Produced by OmniGraffle 7.12.1
+ <dc:date>2020-02-12 20:49:36 +0000</dc:date>
+ </metadata>
+ <g id="Canvas_1" stroke="none" stroke-opacity="1" fill="none" fill-opacity="1" stroke-dasharray="none">
+ <title>Canvas 1</title>
+ <g id="Canvas_1: Layer 1">
+ <title>Layer 1</title>
+ <g id="Graphic_61">
+ <rect x="325" y="1235" width="335" height="342.5" fill="#4751d4" fill-opacity=".04274277"/>
+ <path d="M 325 1235 L 660 1235 L 660 1577.5 L 325 1577.5 Z" stroke="gray" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="1.0,4.0" stroke-width="1"/>
+ <clipPath id="clip_path">
+ <rect x="0" y="0" width="335" height="342.5" fill="#4751d4" fill-opacity=".04274277"/>
+ </clipPath>
+ <text clip-path="url(#clip_path)" transform="translate(330 1240)" fill="black">
+ <tspan font-family="Courier New" font-size="16" font-weight="400" fill="black" x="61.683594" y="13">Worker::waitForWork()</tspan>
+ </text>
+ </g>
+ <g id="Graphic_84" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="390" y="1377.5" width="195" height="45" fill="white"/>
+ <rect x="390" y="1377.5" width="195" height="45" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(396 1385)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="53.804443" y="12">Block Thread</tspan>
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="34.256836" y="27">Until Work Available</tspan>
+ </text>
+ </g>
+ <g id="Graphic_85" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="390" y="1447.5" width="195" height="79.56931" fill="white"/>
+ <rect x="390" y="1447.5" width="195" height="79.56931" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(396 1457.2847)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="13.081055" y="12">Move timed-out fibers from</tspan>
+ <tspan font-family="Courier New" font-size="13" font-weight="700" fill="#515556" x="44.692383" y="26">work.waiting</tspan>
+ <tspan font-family="Roboto" font-size="13" font-weight="400" fill="#515556" x="85.76172" y="42">to</tspan>
+ <tspan font-family="Courier New" font-size="13" font-weight="700" fill="#515556" x="48.59302" y="56">work.fibers</tspan>
+ </text>
+ </g>
+ <g id="Graphic_203" filter="url(#Shadow)">
+ <title>join</title>
+ <rect x="390" y="1307.5" width="195" height="45" fill="#c0c0ff"/>
+ <rect x="390" y="1307.5" width="195" height="45" stroke="#00aeef" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(396 1322.5)" fill="#515556">
+ <tspan font-family="Courier New" font-size="13" font-weight="700" fill="#515556" x="40.79175" y="11">spinForWork()</tspan>
+ </text>
+ </g>
+ <g id="Graphic_204" filter="url(#Shadow)">
+ <path d="M 345 1371.75 L 345 1438.25 C 345 1446.116 336.04 1452.5 325 1452.5 C 313.96 1452.5 305 1446.116 305 1438.25 L 305 1371.75 C 305 1363.884 313.96 1357.5 325 1357.5 C 336.04 1357.5 345 1363.884 345 1371.75 Z" fill="#a7fee5"/>
+ <path d="M 345 1371.75 L 345 1438.25 C 345 1446.116 336.04 1452.5 325 1452.5 C 313.96 1452.5 305 1446.116 305 1438.25 L 305 1371.75 C 305 1363.884 313.96 1357.5 325 1357.5 C 336.04 1357.5 345 1363.884 345 1371.75 Z" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(332.5 1363.5) rotate(90)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="700" fill="#515556" x="27.097168" y="12">Start</tspan>
+ </text>
+ </g>
+ <g id="Graphic_206" filter="url(#Shadow)">
+ <path d="M 680 1371.75 L 680 1438.25 C 680 1446.116 671.04 1452.5 660 1452.5 C 648.96 1452.5 640 1446.116 640 1438.25 L 640 1371.75 C 640 1363.884 648.96 1357.5 660 1357.5 C 671.04 1357.5 680 1363.884 680 1371.75 Z" fill="#ffc7b1"/>
+ <path d="M 680 1371.75 L 680 1438.25 C 680 1446.116 671.04 1452.5 660 1452.5 C 648.96 1452.5 640 1446.116 640 1438.25 L 640 1371.75 C 640 1363.884 648.96 1357.5 660 1357.5 C 671.04 1357.5 680 1363.884 680 1371.75 Z" stroke="#235e00" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
+ <text transform="translate(667.5 1363.5) rotate(90)" fill="#515556">
+ <tspan font-family="Roboto" font-size="13" font-weight="700" fill="#515556" x="26.446533" y="12">Done</tspan>
+ </text>
+ </g>
+ <g id="Line_207">
+ <path d="M 345 1405 L 370 1405 L 370 1277.5 L 487.5 1277.5 L 487.5 1298.44" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_208">
+ <line x1="487.5" y1="1352.5" x2="487.5" y2="1368.44" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_209">
+ <line x1="487.5" y1="1422.5" x2="487.5" y2="1437.44" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ <g id="Line_210">
+ <path d="M 487.5 1527.0693 L 487.5 1552.5 L 610 1552.5 L 610 1405 L 630.94 1405" marker-end="url(#FilledArrow_Marker)" stroke="#00aeef" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="2"/>
+ </g>
+ </g>
+ </g>
+</svg>
diff --git a/docs/scheduler.md b/docs/scheduler.md
new file mode 100644
index 0000000..f272617
--- /dev/null
+++ b/docs/scheduler.md
@@ -0,0 +1,219 @@
+# `marl::Scheduler`
+
+Table of Contents:
+
+- [`marl::Scheduler`](#marlscheduler)
+ - [Binding](#binding)
+ - [Fibers](#fibers)
+ - [Tasks](#tasks)
+ - [Workers](#workers)
+ - [`marl::Scheduler::Worker::run()`](#marlschedulerworkerrun)
+ - [`marl::Scheduler::Worker::runUntilIdle()`](#marlschedulerworkerrununtilidle)
+ - [`marl::Scheduler::Worker::suspend()`](#marlschedulerworkersuspend)
+ - [`marl::Scheduler::Worker::waitForWork()`](#marlschedulerworkerwaitforwork)
+ - [`marl::Scheduler::Worker::spinForWork()`](#marlschedulerworkerspinforwork)
+ - [Worker Types](#worker-types)
+ - [Single-Threaded-Workers](#single-threaded-workers)
+ - [Multi-Threaded-Workers](#multi-threaded-workers)
+
+The `marl::Scheduler` is the most complex part of marl and is responsible for executing tasks and keeping threads running when tasks become blocked.
+
+This document describes the inner workings of the scheduler. This document is not intended to describe usage.
+
+## Binding
+
+The scheduler must be bound to each thread that calls `marl::schedule()`, and unbound from all threads before the scheduler is destructed.
+
+Binding is made using the `marl::Scheduler::bind()` and `marl::Scheduler::unbind()` methods.
+
+Binding assigns a thread-local storage variable so the scheduler is associated with the given thread. This serves two purposes:
+
+1. It allows `marl::schedule()` and the various synchronization primitives to be called without requiring a pointer to the `marl::Scheduler`.
+2. More importantly, it provides a way to get the currently executing fiber for the current thread. This is used by `marl::ConditionVariable::wait()` to suspend the current fiber and place it into a vector so the `marl::ConditionVariable::notify_`xxx`()` methods can reschedule the blocked fibers.
+
+Binding also creates an internal [Single-Threaded-Worker](#single-threaded-workers) for the calling thread. This worker is used for scheduling tasks when there are no [Multi-Threaded-Workers](#multi-threaded-workers) available. Unbinding will ensure that all scheduled tasks for the [Single-Threaded-Worker](#single-threaded-workers) are completed before returning.
+
+## Fibers
+
+A [fiber](https://en.wikipedia.org/wiki/Fiber_(computer_science)) is a lightweight cooperative thread, which can be suspended and resumed at explicit yield points.
+
+At the time of writing, there's no standard and cross-platform library for fibers or coroutines, so marl implements the `marl::OSFiber` class for each supported platform and ABI. Most of these implementations are written in assembly and simply save and restore the callee-saved registers along with maintaining an allocation for the fiber's stack. `marl::OSFiber` is an internal implementation detail, and is not exposed in the public API.
+
+`marl::Scheduler::Fiber` is the public fiber interface that is tightly coupled with the `marl::Scheduler`. The `marl::Scheduler::Fiber` has a simple `std::condition_variable` like interface.
+
+Each `marl::Scheduler::Fiber` is permanently associated with a `marl::Scheduler::Worker`, and is guaranteed to only ever be resumed on the same thread used to suspend.
+
+## Tasks
+
+A `marl::Task` is an alias to `std::function<void()>`, a function that takes no arguments, and returns no value.
+
+Tasks are scheduled using `marl::schedule()`, and are typically implemented as a lambda:
+
+```c++
+marl::schedule([] {
+ printf("Hello world!\n");
+});
+```
+
+While the `marl::Task` signature takes no parameters, it is common to capture variables as part of this lambda for task inputs and outputs.
+
+All the marl synchronization primitives (with exception of `marl::ConditionVariable`) hold a shared pointer to internal state, and you are encouraged to capture these **by value**. This may seem counter-intuitive, but passing by reference can lead to memory corruption if the task outlives the stack used to call `marl::schedule()`. Maintaining a shared state object clearly has allocation and performance overheads, but it was decided that the safety outweighed the costs.
+
+```c++
+marl::WaitGroup wg(1);
+marl::schedule([=] { // capture by value, not reference!
+ printf("Hello world!\n");
+ wg.done();
+});
+wg.wait();
+```
+
+## Workers
+
+The scheduler holds a number of `marl::Scheduler::Worker`s. Each worker holds:
+
+- `work.tasks` - A queue of tasks, yet to be started.
+- `work.fibers` - A queue of suspended fibers, ready to be resumed.
+- `work.waiting` - A queue of suspended fibers, waiting to be resumed or time out.
+- `idleFibers` - A set of idle fibers, ready to be reused.
+- `work.num` - A counter that is kept in sync with `work.tasks.size() + work.fibers.size()`.
+
+When a task is scheduled with a call to `marl::schedule()`, a worker is picked, and the task is placed on to the worker's `work.tasks` queue. The worker is picked using the following rules:
+
+- If the scheduler has no dedicated worker threads (`marl::Scheduler::getWorkerThreadCount() == 0`), then the task is queued on to the [Single-Threaded-Worker](#single-threaded-workers) for the currently executing thread.
+- Otherwise one of the [Multi-Threaded-Workers](#multi-threaded-workers) is picked. If any workers have entered a [spin-for-work](#marlschedulerworkerspinforwork) state, then these will be prioritized, otherwise a [Multi-Threaded-Worker](#multi-threaded-workers) is picked in a round-robin fashion.
+
+### `marl::Scheduler::Worker::run()`
+
+`run()` is the entry point for workers to execute their tasks. The logic is slightly different based on whether the worker is a [Single-Threaded-Worker](#single-threaded-workers) or a [Multi-Threaded-Worker](#multi-threaded-workers), but both share the same basic logic:
+
+- Enter a loop that only exits when the worker is shutdown.
+- In this loop call [`marl::Scheduler::Worker::runUntilIdle()`](#marl::Scheduler::Worker::runUntilIdle())
+
+### `marl::Scheduler::Worker::runUntilIdle()`
+
+As the name suggests, this function executes its work until there is no more work, or all work is blocked.
+
+The basic logic of this method is as follows:
+
+1. Resume any unblocked tasks (fibers)
+
+ `runUntilIdle()` begins by completing all fibers that are ready to be resumed (no longer blocked).
+ This is done by taking a fiber from the `work.fibers` queue, placing the current fiber into the `idleFibers` queue (this fiber is considered idle as it is looking for work), and switching the context over to the taken fiber.
+
+ Executing unblocked fibers is prioritized over starting new tasks. This is because new tasks may result in yet more fibers, and each fiber consumes a certain amount of memory (typically for stack).
+
+2. Start executing new tasks
+
+ Once all resumable fibers have been completed or have become re-blocked, new tasks are taken from the `work.tasks` queue, and are executed. Once a task is completed, control returns back to `runUntilIdle()`, and the main loop starts again from 1.
+
+3. Once there's no more fibers or tasks to execute, `runUntilIdle()` returns.
+
+
+
+### `marl::Scheduler::Worker::suspend()`
+
+Marl allows tasks to block, while keeping threads busy.
+
+If a task blocks, then `Scheduler::Worker::suspend()` is called. `suspend()` begins by calling [`Scheduler::Worker::waitForWork()`](#marl::Scheduler::Worker::waitForWork()), which blocks until there's a task or fiber that can be executed. Then, one of the following occurs:
+
+ 1. If there's any unblocked fibers, the fiber is taken from the `work.fibers` queue and is switched to.
+ 2. If there's any idle fibers, one is taken from the `idleFibers` set and is switched to. This idle fiber when resumed, will continue the role of executing tasks.
+ 3. If none of the above occurs, then a new fiber needs to be created to continue executing tasks. This fiber is created to begin execution in [`marl::Scheduler::Worker::run()`](#marl::Scheduler::Worker::run()), and is switched to.
+
+In all cases, the `suspend()` call switches to another fiber. When the suspended fiber is resumed, `suspend()` returns back to the caller.
+
+
+
+### `marl::Scheduler::Worker::waitForWork()`
+
+When a worker runs out of tasks to start and fibers to resume, `waitForWork()` is called to block until there's something for the worker to do.
+
+If the worker is a [Multi-Threaded-Worker](#multi-threaded-workers), `waitForWork()` begins by entering [`spinForWork()`](#marlschedulerworkerspinforwork), otherwise this stage is skipped.
+
+`waitForWork()` then waits for any of the following to occur before returning:
+
+- A fiber becoming ready to be resumed, by being enqueued on the `work.fibers` queue.
+- A task becoming enqueued on the `work.tasks` queue.
+- A fiber timing out in the `work.waiting` queue.
+- The worker being shutdown.
+
+Any fibers that have timed out in the `work.waiting` queue are automatically moved onto the `work.fibers` queue before returning.
+
+
+
+### `marl::Scheduler::Worker::spinForWork()`
+
+`spinForWork()` has two roles:
+
+1. It attempts to steal work from other workers to keep worker work-loads evenly balanced.
+
+ Task lengths can vary significntly in duration, and over time some workers can end up with a large queue of work, while others are starved. `spinForWork()` is only called when the worker is starved, and will attempt to steal tasks from randomly picked workers. Because fibers must only be executed on the same thread, only tasks, not fibers can be stolen.
+
+2. It attempts to avoid yielding the thread to the OS.
+
+ It is common to have a single task (provider) scheduling many small sub-tasks to the scheduler, which are evenly distributed to the workers (consumers). These consumers typically outnumber the providers, and it is easy to have the provider struggle to provide enough work to keep the consumers fully occupied.
+
+ In this situation, the workers can enter a loop where they are given a task, complete it, and end up waiting a short duration for more work. Allowing a worker thread to yield to the OS when waiting for another task (e.g. with `std::condition_variable::wait()`) can be costly in terms of performance. Depending on the platform, it may take a millisecond or more before the thread is resumed by the OS. A stall of this length can lead to significant stalls in the entire task dependency graph.
+
+`spinForWork()` contains a loop that runs for a short duration. In the body of the loop, the following is performed:
+
+- A tight loop of `nops` is used to keep the CPU busy, while periodically checking `work.num` to see if any new work has become available. If new work is found, `spinForWork()` returns immediately.
+- If no new work was scheduled, an attempt is made to steal a task from another random worker. If the steal was successful, `spinForWork()` returns immediately.
+- If the steal was unsuccessful, `std::this_thread::yield()` is called to prevent marl from starving the OS.
+
+
+
+## Worker Types
+
+A worker is created as either a Single-Threaded-Worker or Multi-Threaded-Worker.
+
+Most of the logic is the same between these two modes.
+The most significant difference between the STW and MTW is the behavior of the worker's entry point function - `marl::Scheduler::Worker::run()`.
+
+### Single-Threaded-Workers
+
+A single-threaded-worker (STW) is created for each thread that is bound with a call to `marl::Scheduler::bind()`.
+
+If the scheduler has no dedicated worker threads (`marl::Scheduler::getWorkerThreadCount() == 0`), then scheduled tasks are queued on to the STW for the currently executing thread.
+
+Because in this mode there are no worker threads, the tasks queued on the STW are not automatically background executed. Instead, tasks are only executed whenever there's a call to [`marl::Scheduler::Worker::suspend()`](#marlschedulerworkersuspend).
+The logic for `suspend()` is common for STWs and MTWs, and the first call will create a fiber which calls `marl::Scheduler::Worker::run()`.
+
+`marl::Scheduler::Worker::run()` is implemented for STWs as a loop that calls `marl::Scheduler::Worker::runUntilIdle()`, and then switches back to the main fiber. This loop only exits once the worker is shutdown.
+
+
+
+```c++
+void SingleThreadedWorkerExample() {
+ marl::Scheduler scheduler;
+ scheduler.setWorkerThreadCount(0); // STW mode.
+ scheduler.bind();
+ defer(scheduler.unbind());
+
+ // Calling marl::schedule() enqueues the task on the STW, but does not
+ // execute it until the thread is blocked.
+ marl::Event done;
+ marl::schedule([=] {
+ done.signal();
+ });
+
+ // This is a blocking call.
+ // marl::Event::wait() (indirectly) calls marl::Scheduler::Worker::suspend().
+ // marl::Scheduler::Worker::suspend() creates and switches to a fiber which
+ // calls marl::Scheduler::Worker::run() to run all enqueued tasks. The fiber
+ // then places itself into idleFibers set and switches back to the main fiber
+ // to continue execution.
+ done.wait();
+}
+```
+
+### Multi-Threaded-Workers
+
+Multi-Threaded-Workers are created when `marl::Scheduler::setWorkerThreadCount()` is called with a positive number.
+
+Each MTW is paired with a new `std::thread` that calls `marl::Scheduler::Worker::run()`.
+
+`marl::Scheduler::Worker::run()` is implemented for MTWs as a loop that calls `marl::Scheduler::Worker::waitForWork()` followed by `marl::Scheduler::Worker::runUntilIdle()`. This loop only exits once the worker is shutdown.
+
+
diff --git a/examples/hello_task.cpp b/examples/hello_task.cpp
index 139406c..6dfff3c 100644
--- a/examples/hello_task.cpp
+++ b/examples/hello_task.cpp
@@ -12,11 +12,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-// Simple "hello world" example that uses marl::Event.
+// Simple "hello world" example that uses marl::Event and marl::WaitGroup.
#include "marl/defer.h"
#include "marl/event.h"
#include "marl/scheduler.h"
+#include "marl/waitgroup.h"
#include <cstdio>
@@ -28,31 +29,36 @@
scheduler.setWorkerThreadCount(4);
defer(scheduler.unbind()); // Automatically unbind before returning.
- // Create an event that automatically resets itself.
- marl::Event sayHellow(marl::Event::Mode::Auto);
- marl::Event saidHellow(marl::Event::Mode::Auto);
+ constexpr int numTasks = 10;
+
+ // Create an event that is manually reset.
+ marl::Event sayHellow(marl::Event::Mode::Manual);
+
+ // Create a WaitGroup with an initial count of numTasks.
+ marl::WaitGroup saidHellow(numTasks);
// Schedule some tasks to run asynchronously.
- for (int i = 0; i < 10; i++) {
+ for (int i = 0; i < numTasks; i++) {
// Each task will run on one of the 4 worker threads.
marl::schedule([=] { // All marl primitives are capture-by-value.
- printf("Task %d waiting to say hello!\n", i);
+ // Decrement the WaitGroup counter when the task has finished.
+ defer(saidHellow.done());
+
+ printf("Task %d waiting to say hello...\n", i);
// Blocking in a task?
// The scheduler will find something else for this thread to do.
sayHellow.wait();
printf("Hello from task %d!\n", i);
-
- saidHellow.signal();
});
}
- // Unblock the tasks one by one.
- for (int i = 0; i < 10; i++) {
- sayHellow.signal();
- saidHellow.wait();
- }
+ sayHellow.signal(); // Unblock all the tasks.
- // All tasks are guaranteed to completed before the scheduler is destructed.
+ saidHellow.wait(); // Wait for all tasks to complete.
+
+ printf("All tasks said hello.\n");
+
+ // All tasks are guaranteed to complete before the scheduler is destructed.
}
diff --git a/include/marl/conditionvariable.h b/include/marl/conditionvariable.h
index daa7c59..2eb7094 100644
--- a/include/marl/conditionvariable.h
+++ b/include/marl/conditionvariable.h
@@ -34,6 +34,8 @@
// thread will work on other tasks until the ConditionVariable is unblocked.
class ConditionVariable {
public:
+ inline ConditionVariable();
+
// notify_one() notifies and potentially unblocks one waiting fiber or thread.
inline void notify_one();
@@ -65,6 +67,11 @@
Predicate&& pred);
private:
+ ConditionVariable(const ConditionVariable&) = delete;
+ ConditionVariable(ConditionVariable&&) = delete;
+ ConditionVariable& operator=(const ConditionVariable&) = delete;
+ ConditionVariable& operator=(ConditionVariable&&) = delete;
+
std::mutex mutex;
std::unordered_set<Scheduler::Fiber*> waiting;
std::condition_variable condition;
@@ -72,6 +79,8 @@
std::atomic<int> numWaitingOnCondition = {0};
};
+ConditionVariable::ConditionVariable() {}
+
void ConditionVariable::notify_one() {
if (numWaiting == 0) {
return;
diff --git a/include/marl/scheduler.h b/include/marl/scheduler.h
index 8f5532d..03d121c 100644
--- a/include/marl/scheduler.h
+++ b/include/marl/scheduler.h
@@ -43,6 +43,7 @@
// A scheduler can be bound to one or more threads using the bind() method.
// Once bound to a thread, that thread can call marl::schedule() to enqueue
// work tasks to be executed asynchronously.
+// All threads must be unbound with unbind() before the scheduler is destructed.
// Scheduler are initially constructed in single-threaded mode.
// Call setWorkerThreadCount() to spawn dedicated worker threads.
class Scheduler {
@@ -53,6 +54,10 @@
using Predicate = std::function<bool()>;
Scheduler(Allocator* allocator = Allocator::Default);
+
+ // Destructor.
+ // Ensure that all threads are unbound before calling - failure to do so may
+ // result in leaked memory.
~Scheduler();
// get() returns the scheduler bound to the current thread.
@@ -64,6 +69,8 @@
// unbind() unbinds the scheduler currently bound to the current thread.
// There must be a existing scheduler bound to the thread prior to calling.
+ // unbind() flushes any enqueued tasks on the single-threaded worker before
+ // returning.
static void unbind();
// enqueue() queues the task for asynchronous execution.
@@ -117,6 +124,7 @@
// will be locked before wait() returns.
// pred will be always be called with the lock held.
// wait() must only be called on the currently executing fiber.
+ _Requires_lock_held_(lock)
void wait(Lock& lock, const Predicate& pred);
// wait() suspends execution of this Fiber until the Fiber is woken up with
@@ -133,6 +141,7 @@
// will be locked before wait() returns.
// pred will be always be called with the lock held.
// wait() must only be called on the currently executing fiber.
+ _Requires_lock_held_(lock)
template <typename Clock, typename Duration>
inline bool wait(Lock& lock,
const std::chrono::time_point<Clock, Duration>& timeout,
@@ -201,6 +210,11 @@
};
private:
+ Scheduler(const Scheduler&) = delete;
+ Scheduler(Scheduler&&) = delete;
+ Scheduler& operator=(const Scheduler&) = delete;
+ Scheduler& operator=(Scheduler&&) = delete;
+
// Stack size in bytes of a new fiber.
// TODO: Make configurable so the default size can be reduced.
static constexpr size_t FiberStackSize = 1024 * 1024;
@@ -271,6 +285,7 @@
// wait() suspends execution of the current task until the predicate pred
// returns true.
// See Fiber::wait() for more information.
+ _Requires_lock_held_(lock)
bool wait(Fiber::Lock& lock,
const TimePoint* timeout,
const Predicate& pred);
@@ -436,6 +451,7 @@
singleThreadedWorkers;
};
+_Requires_lock_held_(lock)
template <typename Clock, typename Duration>
bool Scheduler::Fiber::wait(
Lock& lock,
diff --git a/kokoro/macos/presubmit.sh b/kokoro/macos/presubmit.sh
index da16d77..f2009c8 100755
--- a/kokoro/macos/presubmit.sh
+++ b/kokoro/macos/presubmit.sh
@@ -13,7 +13,7 @@
mkdir build
cd build
- cmake .. -DMARL_BUILD_EXAMPLES=1 -DMARL_BUILD_TESTS=1 -DMARL_WARNINGS_AS_ERRORS=1
+ cmake .. -DMARL_BUILD_EXAMPLES=1 -DMARL_BUILD_TESTS=1 -DMARL_BUILD_BENCHMARKS=1 -DMARL_WARNINGS_AS_ERRORS=1
make -j$(sysctl -n hw.logicalcpu)
./marl-unittests
diff --git a/kokoro/ubuntu/presubmit.sh b/kokoro/ubuntu/presubmit.sh
index 9e19fb0..b6bbe0a 100755
--- a/kokoro/ubuntu/presubmit.sh
+++ b/kokoro/ubuntu/presubmit.sh
@@ -14,7 +14,7 @@
cd build
build_and_run() {
- cmake .. -DMARL_BUILD_EXAMPLES=1 -DMARL_BUILD_TESTS=1 -DMARL_WARNINGS_AS_ERRORS=1 $1
+ cmake .. -DMARL_BUILD_EXAMPLES=1 -DMARL_BUILD_TESTS=1 -DMARL_BUILD_BENCHMARKS=1 -DMARL_WARNINGS_AS_ERRORS=1 $1
make --jobs=$(nproc)
./marl-unittests
diff --git a/kokoro/windows/presubmit.bat b/kokoro/windows/presubmit.bat
index a80c0a3..1cb9caa 100644
--- a/kokoro/windows/presubmit.bat
+++ b/kokoro/windows/presubmit.bat
@@ -20,7 +20,7 @@
if !ERRORLEVEL! neq 0 exit !ERRORLEVEL!
IF /I "%BUILD_SYSTEM%"=="cmake" (
- cmake .. -G "%BUILD_GENERATOR%" "-DMARL_BUILD_TESTS=1" "-DMARL_BUILD_EXAMPLES=1" "-DMARL_WARNINGS_AS_ERRORS=1"
+ cmake .. -G "%BUILD_GENERATOR%" "-DMARL_BUILD_TESTS=1" "-DMARL_BUILD_EXAMPLES=1" "-DMARL_BUILD_BENCHMARKS=1" "-DMARL_WARNINGS_AS_ERRORS=1"
if !ERRORLEVEL! neq 0 exit !ERRORLEVEL!
%MSBUILD% /p:Configuration=%CONFIG% Marl.sln
if !ERRORLEVEL! neq 0 exit !ERRORLEVEL!
diff --git a/src/conditionvariable_test.cpp b/src/conditionvariable_test.cpp
index 3cca0f8..2930e4d 100644
--- a/src/conditionvariable_test.cpp
+++ b/src/conditionvariable_test.cpp
@@ -13,9 +13,13 @@
// limitations under the License.
#include "marl/conditionvariable.h"
+#include "marl/waitgroup.h"
#include "marl_test.h"
+#include <condition_variable>
+#include <mutex>
+
TEST_F(WithoutBoundScheduler, ConditionVariable) {
bool trigger[3] = {false, false, false};
bool signal[3] = {false, false, false};
@@ -25,7 +29,11 @@
std::thread thread([&] {
for (int i = 0; i < 3; i++) {
std::unique_lock<std::mutex> lock(mutex);
- cv.wait(lock, [&] { return trigger[i]; });
+ cv.wait(lock, [&] {
+ EXPECT_TRUE(lock.owns_lock());
+ return trigger[i];
+ });
+ EXPECT_TRUE(lock.owns_lock());
signal[i] = true;
cv.notify_one();
}
@@ -40,7 +48,11 @@
std::unique_lock<std::mutex> lock(mutex);
trigger[i] = true;
cv.notify_one();
- cv.wait(lock, [&] { return signal[i]; });
+ cv.wait(lock, [&] {
+ EXPECT_TRUE(lock.owns_lock());
+ return signal[i];
+ });
+ EXPECT_TRUE(lock.owns_lock());
}
ASSERT_EQ(signal[0], 0 <= i);
@@ -60,7 +72,11 @@
std::thread thread([&] {
for (int i = 0; i < 3; i++) {
std::unique_lock<std::mutex> lock(mutex);
- cv.wait(lock, [&] { return trigger[i]; });
+ cv.wait(lock, [&] {
+ EXPECT_TRUE(lock.owns_lock());
+ return trigger[i];
+ });
+ EXPECT_TRUE(lock.owns_lock());
signal[i] = true;
cv.notify_one();
}
@@ -75,7 +91,11 @@
std::unique_lock<std::mutex> lock(mutex);
trigger[i] = true;
cv.notify_one();
- cv.wait(lock, [&] { return signal[i]; });
+ cv.wait(lock, [&] {
+ EXPECT_TRUE(lock.owns_lock());
+ return signal[i];
+ });
+ EXPECT_TRUE(lock.owns_lock());
}
ASSERT_EQ(signal[0], 0 <= i);
@@ -85,3 +105,39 @@
thread.join();
}
+
+// ConditionVariableTimeouts spins up a whole lot of wait_fors(), unblocking
+// some with timeouts and some with a notify, and then let's all the workers
+// go to idle before repeating.
+// This is testing to ensure that the scheduler handles timeouts correctly when
+// they are early-unblocked, along with expected lock state.
+TEST_P(WithBoundScheduler, ConditionVariableTimeouts) {
+ for (int i = 0; i < 10; i++) {
+ std::mutex mutex;
+ marl::ConditionVariable cv;
+ bool signaled = false; // guarded by mutex
+ auto wg = marl::WaitGroup(100);
+ for (int j = 0; j < 100; j++) {
+ marl::schedule([=, &mutex, &cv, &signaled] {
+ {
+ std::unique_lock<std::mutex> lock(mutex);
+ cv.wait_for(lock, std::chrono::milliseconds(j), [&] {
+ EXPECT_TRUE(lock.owns_lock());
+ return signaled;
+ });
+ EXPECT_TRUE(lock.owns_lock());
+ }
+ // Ensure the mutex unlock happens *before* the wg.done() call,
+ // otherwise the stack pointer may no longer be valid.
+ wg.done();
+ });
+ }
+ std::this_thread::sleep_for(std::chrono::milliseconds(50));
+ {
+ std::unique_lock<std::mutex> lock(mutex);
+ signaled = true;
+ cv.notify_all();
+ }
+ wg.wait();
+ }
+}
diff --git a/src/defer_bench.cpp b/src/defer_bench.cpp
index c89dc32..d826367 100644
--- a/src/defer_bench.cpp
+++ b/src/defer_bench.cpp
@@ -16,10 +16,12 @@
#include "benchmark/benchmark.h"
+volatile int do_not_optimize_away_result = 0;
+
static void Defer(benchmark::State& state) {
- int i = 0;
for (auto _ : state) {
- defer(benchmark::DoNotOptimize(i++));
+ // Avoid benchmark::DoNotOptimize() as this is unfairly slower on Windows.
+ defer(do_not_optimize_away_result++);
}
}
BENCHMARK(Defer);
diff --git a/src/osfiber_ucontext.h b/src/osfiber_ucontext.h
index 664c790..20c3d2b 100644
--- a/src/osfiber_ucontext.h
+++ b/src/osfiber_ucontext.h
@@ -115,6 +115,7 @@
out->target = func;
auto res = getcontext(&out->context);
+ (void)res;
MARL_ASSERT(res == 0, "getcontext() returned %d", int(res));
out->context.uc_stack.ss_sp = out->stack.ptr;
out->context.uc_stack.ss_size = stackSize;
@@ -130,6 +131,7 @@
void OSFiber::switchTo(OSFiber* fiber) {
auto res = swapcontext(&context, &fiber->context);
+ (void)res;
MARL_ASSERT(res == 0, "swapcontext() returned %d", int(res));
}
diff --git a/src/scheduler.cpp b/src/scheduler.cpp
index cae3a6b..197dbb9 100644
--- a/src/scheduler.cpp
+++ b/src/scheduler.cpp
@@ -133,12 +133,17 @@
}
Scheduler::~Scheduler() {
+#if MARL_DEBUG_ENABLED
{
std::unique_lock<std::mutex> lock(singleThreadedWorkerMutex);
MARL_ASSERT(singleThreadedWorkers.size() == 0,
"Scheduler still bound on %d threads",
int(singleThreadedWorkers.size()));
}
+#endif // MARL_DEBUG_ENABLED
+
+ // Release all worker threads.
+ // This will wait for all in-flight tasks to complete before returning.
setWorkerThreadCount(0);
}
@@ -401,6 +406,7 @@
}
}
+_Requires_lock_held_(waitLock)
bool Scheduler::Worker::wait(Fiber::Lock& waitLock,
const TimePoint* timeout,
const Predicate& pred) {
@@ -422,17 +428,20 @@
// Fiber resumed. We don't need the work mutex locked any more.
work.mutex.unlock();
+ // Re-lock to either return due to timeout, or call pred().
+ waitLock.lock();
+
// Check timeout.
if (timeout != nullptr && std::chrono::system_clock::now() >= *timeout) {
return false;
}
- // Spurious wake up. Re-lock, spin again.
- waitLock.lock();
+ // Spurious wake up. Spin again.
}
return true;
}
+_Requires_lock_held_(work.mutex)
void Scheduler::Worker::suspend(
const std::chrono::system_clock::time_point* timeout) {
// Current fiber is yielding as it is blocked.
