GitHub/school-of-sre
1
0
Fork 0
mirror of https://github.com/linkedin/school-of-sre synced 2026-01-07 00:58:03 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
8f118a9c766e15b67b6ea281829e1de8ce826efe
school-of-sre/level102/system_design/resiliency/index.html
github-actions 8f118a9c76 Deployed c42b237 with MkDocs version: 1.2.3
2022-09-25 14:01:28 +00:00

2324 lines
54 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<link rel="canonical" href="https://linkedin.github.io/school-of-sre/level102/system_design/resiliency/">
<link rel="shortcut icon" href="../../../img/favicon.ico">
<meta name="generator" content="mkdocs-1.2.3, mkdocs-material-6.2.8">
<title>Resiliency - School Of SRE</title>
<link rel="stylesheet" href="../../../assets/stylesheets/main.cb6bc1d0.min.css">
<link rel="stylesheet" href="../../../assets/stylesheets/palette.39b8e14a.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700%7CRoboto+Mono&display=fallback">
<style>body,input{font-family:"Roboto",-apple-system,BlinkMacSystemFont,Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono",SFMono-Regular,Consolas,Menlo,monospace}</style>
<link rel="stylesheet" href="../../../stylesheets/custom.css">
</head>
<body dir="ltr" data-md-color-scheme="" data-md-color-primary="none" data-md-color-accent="none">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#quotas" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<script async defer data-domain="linkedin.github.io" src="https://tracking.eskratch.com/js/plausible.js"></script>
<script>document.addEventListener("DOMContentLoaded", function() {
if (document.getElementById("nav-2").checked!=true && document.getElementById("nav-3").checked!=true && window.innerWidth>1220){
document.getElementById("nav-2").checked=true;
document.getElementById("nav-3").checked=true;
}
});</script>
<header class="md-header" data-md-component="header">
<nav class="md-header-nav md-grid" aria-label="Header">
<a href="https://linkedin.github.io/school-of-sre/" title="School Of SRE" class="md-header-nav__button md-logo" aria-label="School Of SRE">
<img src="../../../img/sos.png" alt="logo">
</a>
<!-- Button to open drawer -->
<label class="md-header-nav__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2z"/></svg>
</label>
<div class="md-header-nav__title" data-md-component="header-title">
<div class="md-header-nav__ellipsis">
<span class="md-header-nav__topic md-ellipsis">
<a href="https://linkedin.github.io/school-of-sre/" title="School Of SRE" >
School Of SRE
</a>
</span>
<span class="md-header-nav__topic md-ellipsis">
Resiliency
</span>
</div>
</div>
<label class="md-header-nav__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0116 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 019.5 16 6.5 6.5 0 013 9.5 6.5 6.5 0 019.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5z"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" data-md-state="active" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0116 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 019.5 16 6.5 6.5 0 013 9.5 6.5 6.5 0 019.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5z"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12z"/></svg>
</label>
<button type="reset" class="md-search__icon md-icon" aria-label="Clear" data-md-component="search-reset" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41L17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41z"/></svg>
</button>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
Initializing search
</div>
<ol class="md-search-result__list"></ol>
</div>
</div>
</div>
</div>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../.." class="md-nav__link">
Home
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2" type="checkbox" id="nav-2" >
<label class="md-nav__link" for="nav-2">
Level 101
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Level 101" data-md-level="1">
<label class="md-nav__title" for="nav-2">
<span class="md-nav__icon md-icon"></span>
Level 101
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-1" type="checkbox" id="nav-2-1" >
<label class="md-nav__link" for="nav-2-1">
Fundamentals Series
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Fundamentals Series" data-md-level="2">
<label class="md-nav__title" for="nav-2-1">
<span class="md-nav__icon md-icon"></span>
Fundamentals Series
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-1-1" type="checkbox" id="nav-2-1-1" >
<label class="md-nav__link" for="nav-2-1-1">
Linux Basics
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Linux Basics" data-md-level="3">
<label class="md-nav__title" for="nav-2-1-1">
<span class="md-nav__icon md-icon"></span>
Linux Basics
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/linux_basics/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/linux_basics/command_line_basics/" class="md-nav__link">
Command Line Basics
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/linux_basics/linux_server_administration/" class="md-nav__link">
Server Administration
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/linux_basics/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-1-2" type="checkbox" id="nav-2-1-2" >
<label class="md-nav__link" for="nav-2-1-2">
Git
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Git" data-md-level="3">
<label class="md-nav__title" for="nav-2-1-2">
<span class="md-nav__icon md-icon"></span>
Git
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/git/git-basics/" class="md-nav__link">
Git Basics
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/git/branches/" class="md-nav__link">
Working With Branches
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/git/github-hooks/" class="md-nav__link">
Github and Hooks
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/git/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-1-3" type="checkbox" id="nav-2-1-3" >
<label class="md-nav__link" for="nav-2-1-3">
Linux Networking
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Linux Networking" data-md-level="3">
<label class="md-nav__title" for="nav-2-1-3">
<span class="md-nav__icon md-icon"></span>
Linux Networking
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/linux_networking/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/linux_networking/dns/" class="md-nav__link">
DNS
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/linux_networking/udp/" class="md-nav__link">
UDP
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/linux_networking/http/" class="md-nav__link">
HTTP
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/linux_networking/tcp/" class="md-nav__link">
TCP
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/linux_networking/ipr/" class="md-nav__link">
Routing
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/linux_networking/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-2" type="checkbox" id="nav-2-2" >
<label class="md-nav__link" for="nav-2-2">
Python and Web
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Python and Web" data-md-level="2">
<label class="md-nav__title" for="nav-2-2">
<span class="md-nav__icon md-icon"></span>
Python and Web
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/python_web/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/python_web/python-concepts/" class="md-nav__link">
Some Python Concepts
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/python_web/python-web-flask/" class="md-nav__link">
Python, Web and Flask
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/python_web/url-shorten-app/" class="md-nav__link">
The URL Shortening App
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/python_web/sre-conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-3" type="checkbox" id="nav-2-3" >
<label class="md-nav__link" for="nav-2-3">
Data
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Data" data-md-level="2">
<label class="md-nav__title" for="nav-2-3">
<span class="md-nav__icon md-icon"></span>
Data
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-3-1" type="checkbox" id="nav-2-3-1" >
<label class="md-nav__link" for="nav-2-3-1">
Relational Databases
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Relational Databases" data-md-level="3">
<label class="md-nav__title" for="nav-2-3-1">
<span class="md-nav__icon md-icon"></span>
Relational Databases
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/concepts/" class="md-nav__link">
Key Concepts
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/mysql/" class="md-nav__link">
MySQL
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/innodb/" class="md-nav__link">
InnoDB
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/backup_recovery/" class="md-nav__link">
Backup and Recovery
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/replication/" class="md-nav__link">
MySQL Replication
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-3-1-7" type="checkbox" id="nav-2-3-1-7" >
<label class="md-nav__link" for="nav-2-3-1-7">
Operational Concepts
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Operational Concepts" data-md-level="4">
<label class="md-nav__title" for="nav-2-3-1-7">
<span class="md-nav__icon md-icon"></span>
Operational Concepts
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/select_query/" class="md-nav__link">
Select Query
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/query_performance/" class="md-nav__link">
Query Performance
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/lab/" class="md-nav__link">
Lab
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-3-2" type="checkbox" id="nav-2-3-2" >
<label class="md-nav__link" for="nav-2-3-2">
NoSQL
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="NoSQL" data-md-level="3">
<label class="md-nav__title" for="nav-2-3-2">
<span class="md-nav__icon md-icon"></span>
NoSQL
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/databases_nosql/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_nosql/key_concepts/" class="md-nav__link">
Key Concepts
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_nosql/further_reading/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-3-3" type="checkbox" id="nav-2-3-3" >
<label class="md-nav__link" for="nav-2-3-3">
Big Data
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Big Data" data-md-level="3">
<label class="md-nav__title" for="nav-2-3-3">
<span class="md-nav__icon md-icon"></span>
Big Data
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/big_data/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/big_data/evolution/" class="md-nav__link">
Evolution and Architecture of Hadoop
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/big_data/tasks/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-4" type="checkbox" id="nav-2-4" >
<label class="md-nav__link" for="nav-2-4">
Systems Design
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Systems Design" data-md-level="2">
<label class="md-nav__title" for="nav-2-4">
<span class="md-nav__icon md-icon"></span>
Systems Design
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/systems_design/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/systems_design/scalability/" class="md-nav__link">
Scalability
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/systems_design/availability/" class="md-nav__link">
Availability
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/systems_design/fault-tolerance/" class="md-nav__link">
Fault Tolerance
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/systems_design/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-5" type="checkbox" id="nav-2-5" >
<label class="md-nav__link" for="nav-2-5">
Metrics and Monitoring
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Metrics and Monitoring" data-md-level="2">
<label class="md-nav__title" for="nav-2-5">
<span class="md-nav__icon md-icon"></span>
Metrics and Monitoring
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/metrics_and_monitoring/introduction/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/metrics_and_monitoring/command-line_tools/" class="md-nav__link">
Command-line Tools
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/metrics_and_monitoring/third-party_monitoring/" class="md-nav__link">
Third-party Monitoring
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/metrics_and_monitoring/alerts/" class="md-nav__link">
Proactive Monitoring with Alerts
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/metrics_and_monitoring/best_practices/" class="md-nav__link">
Best Practices for Monitoring
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/metrics_and_monitoring/observability/" class="md-nav__link">
Observability
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/metrics_and_monitoring/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-6" type="checkbox" id="nav-2-6" >
<label class="md-nav__link" for="nav-2-6">
Security
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Security" data-md-level="2">
<label class="md-nav__title" for="nav-2-6">
<span class="md-nav__icon md-icon"></span>
Security
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/security/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/security/fundamentals/" class="md-nav__link">
Fundamentals of Security
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/security/network_security/" class="md-nav__link">
Network Security
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/security/threats_attacks_defences/" class="md-nav__link">
Threat, Attacks & Defences
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/security/writing_secure_code/" class="md-nav__link">
Writing Secure code
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/security/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-3" type="checkbox" id="nav-3" checked>
<label class="md-nav__link" for="nav-3">
Level 102
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Level 102" data-md-level="1">
<label class="md-nav__title" for="nav-3">
<span class="md-nav__icon md-icon"></span>
Level 102
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-3-1" type="checkbox" id="nav-3-1" >
<label class="md-nav__link" for="nav-3-1">
Linux Intermediate
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Linux Intermediate" data-md-level="2">
<label class="md-nav__title" for="nav-3-1">
<span class="md-nav__icon md-icon"></span>
Linux Intermediate
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../linux_intermediate/introduction/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_intermediate/package_management/" class="md-nav__link">
Package Management
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_intermediate/storage_media/" class="md-nav__link">
Storage Media
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_intermediate/archiving_backup/" class="md-nav__link">
Archiving and Backup
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_intermediate/introvim/" class="md-nav__link">
Introduction to Vim
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_intermediate/bashscripting/" class="md-nav__link">
Bash Scripting
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_intermediate/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-3-2" type="checkbox" id="nav-3-2" >
<label class="md-nav__link" for="nav-3-2">
Linux Advanced
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Linux Advanced" data-md-level="2">
<label class="md-nav__title" for="nav-3-2">
<span class="md-nav__icon md-icon"></span>
Linux Advanced
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-3-2-1" type="checkbox" id="nav-3-2-1" >
<label class="md-nav__link" for="nav-3-2-1">
Containerization And Orchestration
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Containerization And Orchestration" data-md-level="3">
<label class="md-nav__title" for="nav-3-2-1">
<span class="md-nav__icon md-icon"></span>
Containerization And Orchestration
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../containerization_and_orchestration/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../containerization_and_orchestration/intro_to_containers/" class="md-nav__link">
Introduction To Containers
</a>
</li>
<li class="md-nav__item">
<a href="../../containerization_and_orchestration/containerization_with_docker/" class="md-nav__link">
Containerization With Docker
</a>
</li>
<li class="md-nav__item">
<a href="../../containerization_and_orchestration/orchestration_with_kubernetes/" class="md-nav__link">
Orchestration With Kubernetes
</a>
</li>
<li class="md-nav__item">
<a href="../../containerization_and_orchestration/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-3-2-2" type="checkbox" id="nav-3-2-2" >
<label class="md-nav__link" for="nav-3-2-2">
System Calls and Signals
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="System Calls and Signals" data-md-level="3">
<label class="md-nav__title" for="nav-3-2-2">
<span class="md-nav__icon md-icon"></span>
System Calls and Signals
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../system_calls_and_signals/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../system_calls_and_signals/signals/" class="md-nav__link">
Signals
</a>
</li>
<li class="md-nav__item">
<a href="../../system_calls_and_signals/system_calls/" class="md-nav__link">
System Calls
</a>
</li>
<li class="md-nav__item">
<a href="../../system_calls_and_signals/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-3-3" type="checkbox" id="nav-3-3" >
<label class="md-nav__link" for="nav-3-3">
Networking
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Networking" data-md-level="2">
<label class="md-nav__title" for="nav-3-3">
<span class="md-nav__icon md-icon"></span>
Networking
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../networking/introduction/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../networking/security/" class="md-nav__link">
Security
</a>
</li>
<li class="md-nav__item">
<a href="../../networking/scale/" class="md-nav__link">
Scale
</a>
</li>
<li class="md-nav__item">
<a href="../../networking/rtt/" class="md-nav__link">
RTT
</a>
</li>
<li class="md-nav__item">
<a href="../../networking/infrastructure-features/" class="md-nav__link">
Infrastructure Services
</a>
</li>
<li class="md-nav__item">
<a href="../../networking/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-3-4" type="checkbox" id="nav-3-4" checked>
<label class="md-nav__link" for="nav-3-4">
System Design
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="System Design" data-md-level="2">
<label class="md-nav__title" for="nav-3-4">
<span class="md-nav__icon md-icon"></span>
System Design
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../large-system-design/" class="md-nav__link">
Large System Design
</a>
</li>
<li class="md-nav__item">
<a href="../scaling/" class="md-nav__link">
Scaling
</a>
</li>
<li class="md-nav__item">
<a href="../scaling-beyond-the-datacenter/" class="md-nav__link">
Scaling Beyond the Data Center
</a>
</li>
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" data-md-toggle="toc" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
Resiliency
<span class="md-nav__icon md-icon"></span>
</label>
<a href="./" class="md-nav__link md-nav__link--active">
Resiliency
</a>
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#quotas" class="md-nav__link">
Quotas
</a>
</li>
<li class="md-nav__item">
<a href="#graceful-degradation" class="md-nav__link">
Graceful Degradation
</a>
</li>
<li class="md-nav__item">
<a href="#timeouts" class="md-nav__link">
Timeouts
</a>
</li>
<li class="md-nav__item">
<a href="#exponential-back-offs" class="md-nav__link">
Exponential back-offs
</a>
</li>
<li class="md-nav__item">
<a href="#circuit-breakers" class="md-nav__link">
Circuit breakers
</a>
</li>
<li class="md-nav__item">
<a href="#self-healing-systems" class="md-nav__link">
Self healing systems
</a>
</li>
<li class="md-nav__item">
<a href="#continuous-deployment-and-integration" class="md-nav__link">
Continuous Deployment and Integration
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-3-5" type="checkbox" id="nav-3-5" >
<label class="md-nav__link" for="nav-3-5">
System Troubleshooting and Performance Improvements
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="System Troubleshooting and Performance Improvements" data-md-level="2">
<label class="md-nav__title" for="nav-3-5">
<span class="md-nav__icon md-icon"></span>
System Troubleshooting and Performance Improvements
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../system_troubleshooting_and_performance/introduction/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../system_troubleshooting_and_performance/troubleshooting/" class="md-nav__link">
Troubleshooting
</a>
</li>
<li class="md-nav__item">
<a href="../../system_troubleshooting_and_performance/important-tools/" class="md-nav__link">
Important Tools
</a>
</li>
<li class="md-nav__item">
<a href="../../system_troubleshooting_and_performance/performance-improvements/" class="md-nav__link">
Performance Improvements
</a>
</li>
<li class="md-nav__item">
<a href="../../system_troubleshooting_and_performance/troubleshooting-example/" class="md-nav__link">
Troubleshooting Example
</a>
</li>
<li class="md-nav__item">
<a href="../../system_troubleshooting_and_performance/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-3-6" type="checkbox" id="nav-3-6" >
<label class="md-nav__link" for="nav-3-6">
Continuous Integration and Continuous Delivery
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Continuous Integration and Continuous Delivery" data-md-level="2">
<label class="md-nav__title" for="nav-3-6">
<span class="md-nav__icon md-icon"></span>
Continuous Integration and Continuous Delivery
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../continuous_integration_and_continuous_delivery/introduction/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../continuous_integration_and_continuous_delivery/introduction_to_cicd/" class="md-nav__link">
What is CI/CD?
</a>
</li>
<li class="md-nav__item">
<a href="../../continuous_integration_and_continuous_delivery/cicd_brief_history/" class="md-nav__link">
Brief History
</a>
</li>
<li class="md-nav__item">
<a href="../../continuous_integration_and_continuous_delivery/continuous_integration_build_pipeline/" class="md-nav__link">
Continuous Integration
</a>
</li>
<li class="md-nav__item">
<a href="../../continuous_integration_and_continuous_delivery/continuous_delivery_release_pipeline/" class="md-nav__link">
Continuous Delivery and Deployment
</a>
</li>
<li class="md-nav__item">
<a href="../../continuous_integration_and_continuous_delivery/jenkins_cicd_pipeline_hands_on_lab/" class="md-nav__link">
CI/CD Pipeline - Hands-on
</a>
</li>
<li class="md-nav__item">
<a href="../../continuous_integration_and_continuous_delivery/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../../../CONTRIBUTING/" class="md-nav__link">
Contribute
</a>
</li>
<li class="md-nav__item">
<a href="../../../CODE_OF_CONDUCT/" class="md-nav__link">
Code of Conduct
</a>
</li>
<li class="md-nav__item">
<a href="../../../sre_community/" class="md-nav__link">
SRE Community
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#quotas" class="md-nav__link">
Quotas
</a>
</li>
<li class="md-nav__item">
<a href="#graceful-degradation" class="md-nav__link">
Graceful Degradation
</a>
</li>
<li class="md-nav__item">
<a href="#timeouts" class="md-nav__link">
Timeouts
</a>
</li>
<li class="md-nav__item">
<a href="#exponential-back-offs" class="md-nav__link">
Exponential back-offs
</a>
</li>
<li class="md-nav__item">
<a href="#circuit-breakers" class="md-nav__link">
Circuit breakers
</a>
</li>
<li class="md-nav__item">
<a href="#self-healing-systems" class="md-nav__link">
Self healing systems
</a>
</li>
<li class="md-nav__item">
<a href="#continuous-deployment-and-integration" class="md-nav__link">
Continuous Deployment and Integration
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content">
<article class="md-content__inner md-typeset">
<h1>Resiliency</h1>
<p>A resilient system is one that can keep functioning in the face of
adversity. With our application, there can be numerous failures that act
as adversities. There can be network level failures that take out entire
data centres, there might be issues at the rack level or at the server
level, or there might be something wrong with the cloud provider. We may also run out of capacity, or there might be a wrong code push that
breaks the system. We will talk about a couple of such issues, and
understand how we might design a system to work around such things. In
some cases, a workaround might not be possible. However it is still
valuable to know potential vulnerabilities to the system stability.</p>
<p>Resilient architectures leverage system design patterns such as
graceful degradation, quotas, timeouts and circuit breakers. Let us look
at some of them in this section.</p>
<h2 id="quotas">Quotas</h2>
<p>A system may have a component or an endpoint that is consumed by
multiple components and endpoints. It is important to have something in
place that will prevent one consumer or client from overwhelming such a
system. Quotas are one way to do this - we simply assign a specific
quota for each component - by way of specifying requests per unit time.
Anyone who breaches the quota is either warned or dropped, depending on
the implementation. This way, one of our own systems misbehaving cannot
result in denial of service to others. Quotas also help us prevent cascading failures.</p>
<h2 id="graceful-degradation">Graceful Degradation</h2>
<p>When a system with multiple dependencies encounters failure in one of
the dependencies, gracefully degrading to minimum viable functionality
would be a lot better than grinding the entire system to a halt. For
example, let us assume there is an endpoint (an URL for a service or a specific function) in our application whose responsibility is to parse the location information in an user uploaded
image from the image's metadata and provide suggestions for location
tagging to the user. Rather than failing the entire upload, it is much
better to skip over this functionality and still give the user an option
to manually tag a location. Gracefully degrading is always better
compared to total failures.</p>
<h2 id="timeouts">Timeouts</h2>
<p>We sometimes call other services or resources like databases or API endpoints in our application. When calling such a resource from our application, it is important to always have a reasonable timeout. It doesn’t necessarily even have to be that the resource will fail for all requests. It just might be that a specific request falls in the high tail latency category. A reasonable time out is helpful to keep the user experience consistent - it is better to fail rather than to have frustratingly long delays, in some cases.</p>
<h2 id="exponential-back-offs">Exponential back-offs</h2>
<p>When a service endpoint fails, retries are one way to see if it was a momentary failure. However, if the retry is also going to fail, there is no point in endlessly retrying. At large enough scale, the retries can compete with the new requests (which might very well be served as expected) and saturate the system. To avoid this, we can look at exponential back-off for retries. This essentially decreases the rate at which the clients retry, upon encountering consecutive failures on retries.</p>
<h2 id="circuit-breakers">Circuit breakers</h2>
<p>While exponential back off is one way to deal with retry storms, circuit breakers can be another. Circuit breakers can help failures from percolating the entire system. Else, an unmitigated failure that flows through the system may result in false alerts, worsening the mean time to detection(MTTD) and mean time to resolution(MTTR). For example, in case one of the in-memory cache nodes fails resulting in requests reaching the database post the initial timeouts for cache, it might end up overloading the database. If the initial connection between cache node failure and DB node failure is not made, then it might result in increased MTTD of the actual cause and consequently the MTTR.</p>
<h2 id="self-healing-systems">Self healing systems</h2>
<p>A traditionally load-balanced application with multiple instances might fail when more than a threshold of instances stop responding to requests - either because they are down, or suddenly there is a huge influx of requests, resulting in degraded performance. A self-healing system adds more instances in this scenario to replace the failed instances.
Auto-scaling like this can also help when there is a sudden spike in query. If our application runs on a public cloud, it might simply be a matter of spinning up more <a href="https://azure.microsoft.com/en-in/overview/what-is-a-virtual-machine/">virtual machines</a>. If we are running on-premise out of our data center, then we will want to think about capacity planning much more carefully. Regardless of how we handle adding additional
capacity - simply addition may not be enough. We should also think about additional potential failure modes that might be encountered. For example, the load balancing layer itself might need scaling up, to handle the influx of new backends.</p>
<h2 id="continuous-deployment-and-integration">Continuous Deployment and Integration</h2>
<p>A well designed system also needs to take into account the need for a proper staging setup that can mimic the production environment as closely as possible. There should also be a way for us to replay production traffic in the staging environment to test changes to production thoroughly.</p>
</article>
</div>
</div>
</main>
<footer class="md-footer">
<div class="md-footer-nav">
<nav class="md-footer-nav__inner md-grid" aria-label="Footer">
<a href="../scaling-beyond-the-datacenter/" class="md-footer-nav__link md-footer-nav__link--prev" rel="prev">
<div class="md-footer-nav__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12z"/></svg>
</div>
<div class="md-footer-nav__title">
<div class="md-ellipsis">
<span class="md-footer-nav__direction">
Previous
</span>
Scaling Beyond the Data Center
</div>
</div>
</a>
<a href="../conclusion/" class="md-footer-nav__link md-footer-nav__link--next" rel="next">
<div class="md-footer-nav__title">
<div class="md-ellipsis">
<span class="md-footer-nav__direction">
Next
</span>
Conclusion
</div>
</div>
<div class="md-footer-nav__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11H4z"/></svg>
</div>
</a>
</nav>
</div>
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-footer-copyright">
<div class="md-footer-copyright__highlight">
Copyright 2020 LinkedIn Corporation. All Rights Reserved. Licensed under the Creative Commons Attribution 4.0 International Public License
</div>
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
<div class="md-footer-social">
<a href="https://github.com/linkedin/school-of-sre" target="_blank" rel="noopener" title="github.com" class="md-footer-social__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
</a>
</div>
</div>
</div>
</footer>
</div>
<script src="../../../assets/javascripts/vendor.18f0862e.min.js"></script>
<script src="../../../assets/javascripts/bundle.994580cf.min.js"></script><script id="__lang" type="application/json">{"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing"}</script>
<script>
app = initialize({
base: "../../..",
features: [],
search: Object.assign({
worker: "../../../assets/javascripts/worker/search.9c0e82ba.min.js"
}, typeof search !== "undefined" && search)
})
</script>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink