GitHub/school-of-sre
1
0
Fork 0
mirror of https://github.com/linkedin/school-of-sre synced 2026-01-08 01:28:03 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
e20d24fb7fe05a4a59ff44ada56969e03c1458bc
school-of-sre/systems_design/fault-tolerance/index.html
Kalyanasundaram Somasundaram e20d24fb7f Deployed a5fc21a with MkDocs version: 1.1.2
2020-11-23 20:43:06 +05:50

1183 lines
30 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<link rel="shortcut icon" href="../../img/favicon.ico">
<meta name="generator" content="mkdocs-1.1.2, mkdocs-material-6.1.5">
<title>Fault Tolerance - SchoolOfSRE</title>
<link rel="stylesheet" href="../../assets/stylesheets/main.21aed14c.min.css">
<link rel="stylesheet" href="../../assets/stylesheets/palette.196e0c26.min.css">
<link href="https://fonts.gstatic.com" rel="preconnect" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700%7CRoboto+Mono&display=fallback">
<style>body,input{font-family:"Roboto",-apple-system,BlinkMacSystemFont,Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono",SFMono-Regular,Consolas,Menlo,monospace}</style>
</head>
<body dir="ltr" data-md-color-scheme="" data-md-color-primary="none" data-md-color-accent="none">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#fault-tolerance" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<header class="md-header" data-md-component="header">
<nav class="md-header-nav md-grid" aria-label="Header">
<a href="../.." title="SchoolOfSRE" class="md-header-nav__button md-logo" aria-label="SchoolOfSRE">
<img src="../../img/sos.png" alt="logo">
</a>
<div class="md-header-nav__title" data-md-component="header-title">
<div class="md-header-nav__ellipsis">
<span class="md-header-nav__topic md-ellipsis">
<a href="../.." title="SchoolOfSRE" >
SchoolOfSRE
</a>
</span>
<span class="md-header-nav__topic md-ellipsis">
Fault Tolerance
</span>
</div>
</div>
<label class="md-header-nav__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0116 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 019.5 16 6.5 6.5 0 013 9.5 6.5 6.5 0 019.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5z"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" data-md-state="active" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0116 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 019.5 16 6.5 6.5 0 013 9.5 6.5 6.5 0 019.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5z"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12z"/></svg>
</label>
<button type="reset" class="md-search__icon md-icon" aria-label="Clear" data-md-component="search-reset" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41L17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41z"/></svg>
</button>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
Initializing search
</div>
<ol class="md-search-result__list"></ol>
</div>
</div>
</div>
</div>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../.." class="md-nav__link">
Home
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2" type="checkbox" id="nav-2" >
<label class="md-nav__link" for="nav-2">
Fundamentals Series
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Fundamentals Series" data-md-level="1">
<label class="md-nav__title" for="nav-2">
<span class="md-nav__icon md-icon"></span>
Fundamentals Series
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-1" type="checkbox" id="nav-2-1" >
<label class="md-nav__link" for="nav-2-1">
Linux Basics
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Linux Basics" data-md-level="2">
<label class="md-nav__title" for="nav-2-1">
<span class="md-nav__icon md-icon"></span>
Linux Basics
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../linux_basics/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_basics/command_line_basics/" class="md-nav__link">
Command Line Basics
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_basics/linux_server_administration/" class="md-nav__link">
Server Administration
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_basics/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-2" type="checkbox" id="nav-2-2" >
<label class="md-nav__link" for="nav-2-2">
Git
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Git" data-md-level="2">
<label class="md-nav__title" for="nav-2-2">
<span class="md-nav__icon md-icon"></span>
Git
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../git/git-basics/" class="md-nav__link">
Git Basics
</a>
</li>
<li class="md-nav__item">
<a href="../../git/branches/" class="md-nav__link">
Working With Branches
</a>
</li>
<li class="md-nav__item">
<a href="../../git/github-hooks/" class="md-nav__link">
Github and Hooks
</a>
</li>
<li class="md-nav__item">
<a href="../../git/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-3" type="checkbox" id="nav-2-3" >
<label class="md-nav__link" for="nav-2-3">
Linux Networking
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Linux Networking" data-md-level="2">
<label class="md-nav__title" for="nav-2-3">
<span class="md-nav__icon md-icon"></span>
Linux Networking
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../linux_networking/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_networking/dns/" class="md-nav__link">
DNS
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_networking/udp/" class="md-nav__link">
UDP
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_networking/http/" class="md-nav__link">
HTTP
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_networking/tcp/" class="md-nav__link">
TCP
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_networking/ipr/" class="md-nav__link">
Routing
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_networking/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-3" type="checkbox" id="nav-3" >
<label class="md-nav__link" for="nav-3">
Python and Web
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Python and Web" data-md-level="1">
<label class="md-nav__title" for="nav-3">
<span class="md-nav__icon md-icon"></span>
Python and Web
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../python_web/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../python_web/python-concepts/" class="md-nav__link">
Some Python Concepts
</a>
</li>
<li class="md-nav__item">
<a href="../../python_web/python-web-flask/" class="md-nav__link">
Python, Web and Flask
</a>
</li>
<li class="md-nav__item">
<a href="../../python_web/url-shorten-app/" class="md-nav__link">
The URL Shortening App
</a>
</li>
<li class="md-nav__item">
<a href="../../python_web/sre-conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-4" type="checkbox" id="nav-4" >
<label class="md-nav__link" for="nav-4">
Data
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Data" data-md-level="1">
<label class="md-nav__title" for="nav-4">
<span class="md-nav__icon md-icon"></span>
Data
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-4-1" type="checkbox" id="nav-4-1" >
<label class="md-nav__link" for="nav-4-1">
NoSQL Concepts
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="NoSQL Concepts" data-md-level="2">
<label class="md-nav__title" for="nav-4-1">
<span class="md-nav__icon md-icon"></span>
NoSQL Concepts
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../databases_nosql/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../databases_nosql/key_concepts/" class="md-nav__link">
Key Concepts
</a>
</li>
<li class="md-nav__item">
<a href="../../databases_nosql/further_reading/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-4-2" type="checkbox" id="nav-4-2" >
<label class="md-nav__link" for="nav-4-2">
Big Data
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Big Data" data-md-level="2">
<label class="md-nav__title" for="nav-4-2">
<span class="md-nav__icon md-icon"></span>
Big Data
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../big_data/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../big_data/evolution/" class="md-nav__link">
Evolution and Architecure of Hadoop
</a>
</li>
<li class="md-nav__item">
<a href="../../big_data/tasks/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-5" type="checkbox" id="nav-5" checked>
<label class="md-nav__link" for="nav-5">
Systems Design
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Systems Design" data-md-level="1">
<label class="md-nav__title" for="nav-5">
<span class="md-nav__icon md-icon"></span>
Systems Design
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../scalability/" class="md-nav__link">
Scalability
</a>
</li>
<li class="md-nav__item">
<a href="../availability/" class="md-nav__link">
Availability
</a>
</li>
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" data-md-toggle="toc" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
Fault Tolerance
<span class="md-nav__icon md-icon"></span>
</label>
<a href="./" class="md-nav__link md-nav__link--active">
Fault Tolerance
</a>
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#fault-tolerance-failure-metrics" class="md-nav__link">
Fault Tolerance - Failure Metrics
</a>
<nav class="md-nav" aria-label="Fault Tolerance - Failure Metrics">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#refer" class="md-nav__link">
Refer
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#fault-tolerance-fault-isolation-terms" class="md-nav__link">
Fault Tolerance - Fault Isolation Terms
</a>
</li>
<li class="md-nav__item">
<a href="#swimlane-principles" class="md-nav__link">
Swimlane Principles
</a>
</li>
<li class="md-nav__item">
<a href="#swimlane-approaches" class="md-nav__link">
Swimlane Approaches
</a>
<nav class="md-nav" aria-label="Swimlane Approaches">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#refer_1" class="md-nav__link">
Refer
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#applications-in-sre-role" class="md-nav__link">
Applications in SRE role
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-6" type="checkbox" id="nav-6" >
<label class="md-nav__link" for="nav-6">
Security
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Security" data-md-level="1">
<label class="md-nav__title" for="nav-6">
<span class="md-nav__icon md-icon"></span>
Security
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../security/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../security/fundamentals/" class="md-nav__link">
Fundamentals of Security
</a>
</li>
<li class="md-nav__item">
<a href="../../security/network_security/" class="md-nav__link">
Network Security
</a>
</li>
<li class="md-nav__item">
<a href="../../security/threats_attacks_defences/" class="md-nav__link">
Threat, Attacks & Defences
</a>
</li>
<li class="md-nav__item">
<a href="../../security/writing_secure_code/" class="md-nav__link">
Writing Secure code
</a>
</li>
<li class="md-nav__item">
<a href="../../security/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../../CONTRIBUTING/" class="md-nav__link">
Contribute
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#fault-tolerance-failure-metrics" class="md-nav__link">
Fault Tolerance - Failure Metrics
</a>
<nav class="md-nav" aria-label="Fault Tolerance - Failure Metrics">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#refer" class="md-nav__link">
Refer
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#fault-tolerance-fault-isolation-terms" class="md-nav__link">
Fault Tolerance - Fault Isolation Terms
</a>
</li>
<li class="md-nav__item">
<a href="#swimlane-principles" class="md-nav__link">
Swimlane Principles
</a>
</li>
<li class="md-nav__item">
<a href="#swimlane-approaches" class="md-nav__link">
Swimlane Approaches
</a>
<nav class="md-nav" aria-label="Swimlane Approaches">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#refer_1" class="md-nav__link">
Refer
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#applications-in-sre-role" class="md-nav__link">
Applications in SRE role
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content">
<article class="md-content__inner md-typeset">
<h1 id="fault-tolerance">Fault Tolerance</h1>
<p>Failures are not avoidable in any system and will happen all the time, hence we need to build systems that can tolerate failures or recover from them.</p>
<ul>
<li>In systems, failure is the norm rather than the exception.</li>
<li>"Anything that can go wrong will go wrong” -- Murphy’s Law</li>
<li>“Complex systems contain changing mixtures of failures latent within them” -- How Complex Systems Fail.</li>
</ul>
<h3 id="fault-tolerance-failure-metrics">Fault Tolerance - Failure Metrics</h3>
<p>Common failure metrics that get measured and tracked for any system.</p>
<p><strong>Mean time to repair (MTTR):</strong> The average time to repair and restore a failed system. </p>
<p><strong>Mean time between failures (MTBF):</strong> The average operational time between one device failure or system breakdown and the next. </p>
<p><strong>Mean time to failure (MTTF):</strong> The average time a device or system is expected to function before it fails. </p>
<p><strong>Mean time to detect (MTTD):</strong> The average time between the onset of a problem and when the organization detects it. </p>
<p><strong>Mean time to investigate (MTTI):</strong> The average time between the detection of an incident and when the organization begins to investigate its cause and solution. </p>
<p><strong>Mean time to restore service (MTRS):</strong> The average elapsed time from the detection of an incident until the affected system or component is again available to users.</p>
<p><strong>Mean time between system incidents (MTBSI):</strong> The average elapsed time between the detection of two consecutive incidents. MTBSI can be calculated by adding MTBF and MTRS (MTBSI = MTBF + MTRS).</p>
<p><strong>Failure rate:</strong> Another reliability metric, which measures the frequency with which a component or system fails. It is expressed as a number of failures over a unit of time.</p>
<h4 id="refer">Refer</h4>
<ul>
<li>https://www.splunk.com/en_us/data-insider/what-is-mean-time-to-repair.html</li>
</ul>
<h3 id="fault-tolerance-fault-isolation-terms">Fault Tolerance - Fault Isolation Terms</h3>
<p>Systems should have a short circuit. Say in our content sharing system, if “Notifications” is not working, the site should gracefully handle that failure by removing the functionality instead of taking the whole site down. </p>
<p>Swimlane is one of the commonly used fault isolation methodology. Swimlane adds a barrier to the service from other services so that failure on either of them won’t affect the other. Say we roll out a new feature ‘Advertisement’ in our content sharing app.
We can have two architectures
<img alt="Swimlane" src="../images/swimlane-1.jpg" /></p>
<p>If Ads are generated on the fly synchronously during each Newsfeed request, the faults in Ads feature gets propagated to Newsfeed feature. Instead if we swimlane “Generation of Ads” service and use a shared storage to populate Newsfeed App, Ads failures won’t cascade to Newsfeed and worst case if Ads don’t meet SLA , we can have Newsfeed without Ads.</p>
<p>Let's take another example, we come up with a new model for our Content sharing App. Here we roll out enterprise content sharing App where enterprises pay for the service and the content should never be shared outside the enterprise. </p>
<p><img alt="Swimlane-principles" src="../images/swimlane-2.jpg" /></p>
<h3 id="swimlane-principles">Swimlane Principles</h3>
<p><strong>Principle 1:</strong> Nothing is shared (also known as “share as little as possible”). The less that is shared within a swim lane, the more fault isolative the swim lane becomes. (as shown in Enterprise usecase)</p>
<p><strong>Principle 2:</strong> Nothing crosses a swim lane boundary. Synchronous (defined by expecting a request—not the transfer protocol) communication never crosses a swim lane boundary; if it does, the boundary is drawn incorrectly. (as shown in Ads feature)</p>
<h3 id="swimlane-approaches">Swimlane Approaches</h3>
<p><strong>Approach 1:</strong> Swim lane the money-maker. Never allow your cash register to be compromised by other systems. (Tier 1 vs Tier 2 in enterprise use case)</p>
<p><strong>Approach 2:</strong> Swim lane the biggest sources of incidents. Identify the recurring causes of pain and isolate them.(if Ads feature is in code yellow, swim laning it is the best option)</p>
<p><strong>Approach 3:</strong> Swim lane natural barriers. Customer boundaries make good swim lanes.(Public vs Enterprise customers)</p>
<h4 id="refer_1">Refer</h4>
<ul>
<li>https://learning.oreilly.com/library/view/the-art-of/9780134031408/ch21.html#ch21</li>
</ul>
<h3 id="applications-in-sre-role">Applications in SRE role</h3>
<ol>
<li>Work with the DC tech or cloud team to distribute infrastructure such that its immune to switch or power failures by creating fault zones within a Data Center
https://docs.microsoft.com/en-us/azure/virtual-machines/manage-availability#use-availability-zones-to-protect-from-datacenter-level-failures</li>
<li>Work with the partners and design interaction between services such that one service breakdown is not amplified in a cascading fashion to all upstreams</li>
</ol>
</article>
</div>
</div>
</main>
<footer class="md-footer">
<div class="md-footer-nav">
<nav class="md-footer-nav__inner md-grid" aria-label="Footer">
<a href="../availability/" class="md-footer-nav__link md-footer-nav__link--prev" rel="prev">
<div class="md-footer-nav__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12z"/></svg>
</div>
<div class="md-footer-nav__title">
<div class="md-ellipsis">
<span class="md-footer-nav__direction">
Previous
</span>
Availability
</div>
</div>
</a>
<a href="../conclusion/" class="md-footer-nav__link md-footer-nav__link--next" rel="next">
<div class="md-footer-nav__title">
<div class="md-ellipsis">
<span class="md-footer-nav__direction">
Next
</span>
Conclusion
</div>
</div>
<div class="md-footer-nav__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11H4z"/></svg>
</div>
</a>
</nav>
</div>
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-footer-copyright">
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
</div>
</div>
</footer>
</div>
<script src="../../assets/javascripts/vendor.7e0ee788.min.js"></script>
<script src="../../assets/javascripts/bundle.b3a72adc.min.js"></script><script id="__lang" type="application/json">{"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing"}</script>
<script>
app = initialize({
base: "../..",
features: [],
search: Object.assign({
worker: "../../assets/javascripts/worker/search.4ac00218.min.js"
}, typeof search !== "undefined" && search)
})
</script>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink