GitHub/school-of-sre
1
0
Fork 0
mirror of https://github.com/linkedin/school-of-sre synced 2026-01-18 22:48:03 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
ad2c2b1c8bbca21e100ab297f9953cbb9e95178f
school-of-sre/level102/system_troubleshooting_and_performance/troubleshooting-example/index.html
github-actions ad2c2b1c8b Deployed a4319ee with MkDocs version: 1.1.2
2021-08-13 10:31:27 +00:00

2228 lines
52 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<link rel="canonical" href="https://linkedin.github.io/school-of-sre/level102/system_troubleshooting_and_performance/troubleshooting-example/">
<link rel="shortcut icon" href="../../../img/favicon.ico">
<meta name="generator" content="mkdocs-1.1.2, mkdocs-material-6.2.8">
<title>Troubleshooting Example - School Of SRE</title>
<link rel="stylesheet" href="../../../assets/stylesheets/main.cb6bc1d0.min.css">
<link rel="stylesheet" href="../../../assets/stylesheets/palette.39b8e14a.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700%7CRoboto+Mono&display=fallback">
<style>body,input{font-family:"Roboto",-apple-system,BlinkMacSystemFont,Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono",SFMono-Regular,Consolas,Menlo,monospace}</style>
<link rel="stylesheet" href="../../../stylesheets/custom.css">
</head>
<body dir="ltr" data-md-color-scheme="" data-md-color-primary="none" data-md-color-accent="none">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#example-memory-leak" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<script async defer data-domain="linkedin.github.io" src="https://tracking.eskratch.com/js/plausible.js"></script>
<script>document.addEventListener("DOMContentLoaded", function() {
if (document.getElementById("nav-2").checked!=true && document.getElementById("nav-3").checked!=true && window.innerWidth>1220){
document.getElementById("nav-2").checked=true;
document.getElementById("nav-3").checked=true;
}
});</script>
<header class="md-header" data-md-component="header">
<nav class="md-header-nav md-grid" aria-label="Header">
<a href="https://linkedin.github.io/school-of-sre/" title="School Of SRE" class="md-header-nav__button md-logo" aria-label="School Of SRE">
<img src="../../../img/sos.png" alt="logo">
</a>
<!-- Button to open drawer -->
<label class="md-header-nav__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2z"/></svg>
</label>
<div class="md-header-nav__title" data-md-component="header-title">
<div class="md-header-nav__ellipsis">
<span class="md-header-nav__topic md-ellipsis">
<a href="https://linkedin.github.io/school-of-sre/" title="School Of SRE" >
School Of SRE
</a>
</span>
<span class="md-header-nav__topic md-ellipsis">
Troubleshooting Example
</span>
</div>
</div>
<label class="md-header-nav__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0116 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 019.5 16 6.5 6.5 0 013 9.5 6.5 6.5 0 019.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5z"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" data-md-state="active" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0116 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 019.5 16 6.5 6.5 0 013 9.5 6.5 6.5 0 019.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5z"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12z"/></svg>
</label>
<button type="reset" class="md-search__icon md-icon" aria-label="Clear" data-md-component="search-reset" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41L17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41z"/></svg>
</button>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
Initializing search
</div>
<ol class="md-search-result__list"></ol>
</div>
</div>
</div>
</div>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../.." class="md-nav__link">
Home
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2" type="checkbox" id="nav-2" >
<label class="md-nav__link" for="nav-2">
Level 101
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Level 101" data-md-level="1">
<label class="md-nav__title" for="nav-2">
<span class="md-nav__icon md-icon"></span>
Level 101
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-1" type="checkbox" id="nav-2-1" >
<label class="md-nav__link" for="nav-2-1">
Fundamentals Series
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Fundamentals Series" data-md-level="2">
<label class="md-nav__title" for="nav-2-1">
<span class="md-nav__icon md-icon"></span>
Fundamentals Series
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-1-1" type="checkbox" id="nav-2-1-1" >
<label class="md-nav__link" for="nav-2-1-1">
Linux Basics
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Linux Basics" data-md-level="3">
<label class="md-nav__title" for="nav-2-1-1">
<span class="md-nav__icon md-icon"></span>
Linux Basics
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/linux_basics/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/linux_basics/command_line_basics/" class="md-nav__link">
Command Line Basics
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/linux_basics/linux_server_administration/" class="md-nav__link">
Server Administration
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/linux_basics/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-1-2" type="checkbox" id="nav-2-1-2" >
<label class="md-nav__link" for="nav-2-1-2">
Git
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Git" data-md-level="3">
<label class="md-nav__title" for="nav-2-1-2">
<span class="md-nav__icon md-icon"></span>
Git
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/git/git-basics/" class="md-nav__link">
Git Basics
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/git/branches/" class="md-nav__link">
Working With Branches
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/git/github-hooks/" class="md-nav__link">
Github and Hooks
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/git/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-1-3" type="checkbox" id="nav-2-1-3" >
<label class="md-nav__link" for="nav-2-1-3">
Linux Networking
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Linux Networking" data-md-level="3">
<label class="md-nav__title" for="nav-2-1-3">
<span class="md-nav__icon md-icon"></span>
Linux Networking
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/linux_networking/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/linux_networking/dns/" class="md-nav__link">
DNS
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/linux_networking/udp/" class="md-nav__link">
UDP
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/linux_networking/http/" class="md-nav__link">
HTTP
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/linux_networking/tcp/" class="md-nav__link">
TCP
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/linux_networking/ipr/" class="md-nav__link">
Routing
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/linux_networking/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-2" type="checkbox" id="nav-2-2" >
<label class="md-nav__link" for="nav-2-2">
Python and Web
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Python and Web" data-md-level="2">
<label class="md-nav__title" for="nav-2-2">
<span class="md-nav__icon md-icon"></span>
Python and Web
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/python_web/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/python_web/python-concepts/" class="md-nav__link">
Some Python Concepts
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/python_web/python-web-flask/" class="md-nav__link">
Python, Web and Flask
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/python_web/url-shorten-app/" class="md-nav__link">
The URL Shortening App
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/python_web/sre-conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-3" type="checkbox" id="nav-2-3" >
<label class="md-nav__link" for="nav-2-3">
Data
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Data" data-md-level="2">
<label class="md-nav__title" for="nav-2-3">
<span class="md-nav__icon md-icon"></span>
Data
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-3-1" type="checkbox" id="nav-2-3-1" >
<label class="md-nav__link" for="nav-2-3-1">
Relational Databases
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Relational Databases" data-md-level="3">
<label class="md-nav__title" for="nav-2-3-1">
<span class="md-nav__icon md-icon"></span>
Relational Databases
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/concepts/" class="md-nav__link">
Key Concepts
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/mysql/" class="md-nav__link">
MySQL
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/innodb/" class="md-nav__link">
InnoDB
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/backup_recovery/" class="md-nav__link">
Backup and Recovery
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/replication/" class="md-nav__link">
MySQL Replication
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-3-1-7" type="checkbox" id="nav-2-3-1-7" >
<label class="md-nav__link" for="nav-2-3-1-7">
Operational Concepts
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Operational Concepts" data-md-level="4">
<label class="md-nav__title" for="nav-2-3-1-7">
<span class="md-nav__icon md-icon"></span>
Operational Concepts
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/select_query/" class="md-nav__link">
Select Query
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/query_performance/" class="md-nav__link">
Query Performance
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/lab/" class="md-nav__link">
Lab
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_sql/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-3-2" type="checkbox" id="nav-2-3-2" >
<label class="md-nav__link" for="nav-2-3-2">
NoSQL
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="NoSQL" data-md-level="3">
<label class="md-nav__title" for="nav-2-3-2">
<span class="md-nav__icon md-icon"></span>
NoSQL
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/databases_nosql/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_nosql/key_concepts/" class="md-nav__link">
Key Concepts
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/databases_nosql/further_reading/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-3-3" type="checkbox" id="nav-2-3-3" >
<label class="md-nav__link" for="nav-2-3-3">
Big Data
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Big Data" data-md-level="3">
<label class="md-nav__title" for="nav-2-3-3">
<span class="md-nav__icon md-icon"></span>
Big Data
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/big_data/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/big_data/evolution/" class="md-nav__link">
Evolution and Architecture of Hadoop
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/big_data/tasks/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-4" type="checkbox" id="nav-2-4" >
<label class="md-nav__link" for="nav-2-4">
Systems Design
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Systems Design" data-md-level="2">
<label class="md-nav__title" for="nav-2-4">
<span class="md-nav__icon md-icon"></span>
Systems Design
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/systems_design/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/systems_design/scalability/" class="md-nav__link">
Scalability
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/systems_design/availability/" class="md-nav__link">
Availability
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/systems_design/fault-tolerance/" class="md-nav__link">
Fault Tolerance
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/systems_design/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-5" type="checkbox" id="nav-2-5" >
<label class="md-nav__link" for="nav-2-5">
Metrics and Monitoring
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Metrics and Monitoring" data-md-level="2">
<label class="md-nav__title" for="nav-2-5">
<span class="md-nav__icon md-icon"></span>
Metrics and Monitoring
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/metrics_and_monitoring/introduction/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/metrics_and_monitoring/command-line_tools/" class="md-nav__link">
Command-line Tools
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/metrics_and_monitoring/third-party_monitoring/" class="md-nav__link">
Third-party Monitoring
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/metrics_and_monitoring/alerts/" class="md-nav__link">
Proactive Monitoring with Alerts
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/metrics_and_monitoring/best_practices/" class="md-nav__link">
Best Practices for Monitoring
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/metrics_and_monitoring/observability/" class="md-nav__link">
Observability
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/metrics_and_monitoring/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-2-6" type="checkbox" id="nav-2-6" >
<label class="md-nav__link" for="nav-2-6">
Security
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Security" data-md-level="2">
<label class="md-nav__title" for="nav-2-6">
<span class="md-nav__icon md-icon"></span>
Security
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../level101/security/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/security/fundamentals/" class="md-nav__link">
Fundamentals of Security
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/security/network_security/" class="md-nav__link">
Network Security
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/security/threats_attacks_defences/" class="md-nav__link">
Threat, Attacks & Defences
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/security/writing_secure_code/" class="md-nav__link">
Writing Secure code
</a>
</li>
<li class="md-nav__item">
<a href="../../../level101/security/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-3" type="checkbox" id="nav-3" checked>
<label class="md-nav__link" for="nav-3">
Level 102
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Level 102" data-md-level="1">
<label class="md-nav__title" for="nav-3">
<span class="md-nav__icon md-icon"></span>
Level 102
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-3-1" type="checkbox" id="nav-3-1" >
<label class="md-nav__link" for="nav-3-1">
Linux Intermediate
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Linux Intermediate" data-md-level="2">
<label class="md-nav__title" for="nav-3-1">
<span class="md-nav__icon md-icon"></span>
Linux Intermediate
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../linux_intermediate/introduction/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_intermediate/package_management/" class="md-nav__link">
Package Management
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_intermediate/storage_media/" class="md-nav__link">
Storage Media
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_intermediate/archiving_backup/" class="md-nav__link">
Archiving and Backup
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_intermediate/introvim/" class="md-nav__link">
Introduction to Vim
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_intermediate/bashscripting/" class="md-nav__link">
Bash Scripting
</a>
</li>
<li class="md-nav__item">
<a href="../../linux_intermediate/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-3-2" type="checkbox" id="nav-3-2" >
<label class="md-nav__link" for="nav-3-2">
Linux Advanced
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Linux Advanced" data-md-level="2">
<label class="md-nav__title" for="nav-3-2">
<span class="md-nav__icon md-icon"></span>
Linux Advanced
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-3-2-1" type="checkbox" id="nav-3-2-1" >
<label class="md-nav__link" for="nav-3-2-1">
Containerization And Orchestration
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Containerization And Orchestration" data-md-level="3">
<label class="md-nav__title" for="nav-3-2-1">
<span class="md-nav__icon md-icon"></span>
Containerization And Orchestration
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../containerization_and_orchestration/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../containerization_and_orchestration/intro_to_containers/" class="md-nav__link">
Introduction To Containers
</a>
</li>
<li class="md-nav__item">
<a href="../../containerization_and_orchestration/containerization_with_docker/" class="md-nav__link">
Containerization With Docker
</a>
</li>
<li class="md-nav__item">
<a href="../../containerization_and_orchestration/orchestration_with_kubernetes/" class="md-nav__link">
Orchestration With Kubernetes
</a>
</li>
<li class="md-nav__item">
<a href="../../containerization_and_orchestration/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-3-2-2" type="checkbox" id="nav-3-2-2" >
<label class="md-nav__link" for="nav-3-2-2">
System Calls and Signals
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="System Calls and Signals" data-md-level="3">
<label class="md-nav__title" for="nav-3-2-2">
<span class="md-nav__icon md-icon"></span>
System Calls and Signals
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../system_calls_and_signals/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../system_calls_and_signals/signals/" class="md-nav__link">
Signals
</a>
</li>
<li class="md-nav__item">
<a href="../../system_calls_and_signals/system_calls/" class="md-nav__link">
System Calls
</a>
</li>
<li class="md-nav__item">
<a href="../../system_calls_and_signals/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-3-3" type="checkbox" id="nav-3-3" >
<label class="md-nav__link" for="nav-3-3">
Networking
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Networking" data-md-level="2">
<label class="md-nav__title" for="nav-3-3">
<span class="md-nav__icon md-icon"></span>
Networking
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../networking/introduction/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../networking/security/" class="md-nav__link">
Security
</a>
</li>
<li class="md-nav__item">
<a href="../../networking/scale/" class="md-nav__link">
Scale
</a>
</li>
<li class="md-nav__item">
<a href="../../networking/rtt/" class="md-nav__link">
RTT
</a>
</li>
<li class="md-nav__item">
<a href="../../networking/infrastructure-features/" class="md-nav__link">
Infrastructure Services
</a>
</li>
<li class="md-nav__item">
<a href="../../networking/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-3-4" type="checkbox" id="nav-3-4" >
<label class="md-nav__link" for="nav-3-4">
System Design
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="System Design" data-md-level="2">
<label class="md-nav__title" for="nav-3-4">
<span class="md-nav__icon md-icon"></span>
System Design
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../system_design/intro/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../system_design/large-system-design/" class="md-nav__link">
Large System Design
</a>
</li>
<li class="md-nav__item">
<a href="../../system_design/scaling/" class="md-nav__link">
Scaling
</a>
</li>
<li class="md-nav__item">
<a href="../../system_design/scaling-beyond-the-datacenter/" class="md-nav__link">
Scaling Beyond the Data Center
</a>
</li>
<li class="md-nav__item">
<a href="../../system_design/resiliency/" class="md-nav__link">
Resiliency
</a>
</li>
<li class="md-nav__item">
<a href="../../system_design/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-3-5" type="checkbox" id="nav-3-5" checked>
<label class="md-nav__link" for="nav-3-5">
System Troubleshooting and Performance Improvements
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="System Troubleshooting and Performance Improvements" data-md-level="2">
<label class="md-nav__title" for="nav-3-5">
<span class="md-nav__icon md-icon"></span>
System Troubleshooting and Performance Improvements
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../introduction/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../troubleshooting/" class="md-nav__link">
Troubleshooting
</a>
</li>
<li class="md-nav__item">
<a href="../important-tools/" class="md-nav__link">
Important Tools
</a>
</li>
<li class="md-nav__item">
<a href="../performance-improvements/" class="md-nav__link">
Performance Improvements
</a>
</li>
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" data-md-toggle="toc" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
Troubleshooting Example
<span class="md-nav__icon md-icon"></span>
</label>
<a href="./" class="md-nav__link md-nav__link--active">
Troubleshooting Example
</a>
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#example-memory-leak" class="md-nav__link">
Example - Memory leak :
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="nav-3-6" type="checkbox" id="nav-3-6" >
<label class="md-nav__link" for="nav-3-6">
Continuous Integration and Continuous Delivery
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Continuous Integration and Continuous Delivery" data-md-level="2">
<label class="md-nav__title" for="nav-3-6">
<span class="md-nav__icon md-icon"></span>
Continuous Integration and Continuous Delivery
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../continuous_integration_and_continuous_delivery/introduction/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../../continuous_integration_and_continuous_delivery/introduction_to_cicd/" class="md-nav__link">
What is CI/CD?
</a>
</li>
<li class="md-nav__item">
<a href="../../continuous_integration_and_continuous_delivery/cicd_brief_history/" class="md-nav__link">
Brief History
</a>
</li>
<li class="md-nav__item">
<a href="../../continuous_integration_and_continuous_delivery/continuous_integration_build_pipeline/" class="md-nav__link">
Continuous Integration
</a>
</li>
<li class="md-nav__item">
<a href="../../continuous_integration_and_continuous_delivery/continuous_delivery_release_pipeline/" class="md-nav__link">
Continuous Delivery and Deployment
</a>
</li>
<li class="md-nav__item">
<a href="../../continuous_integration_and_continuous_delivery/jenkins_cicd_pipeline_hands_on_lab/" class="md-nav__link">
CI/CD Pipeline - Hands-on
</a>
</li>
<li class="md-nav__item">
<a href="../../continuous_integration_and_continuous_delivery/conclusion/" class="md-nav__link">
Conclusion
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../../../CONTRIBUTING/" class="md-nav__link">
Contribute
</a>
</li>
<li class="md-nav__item">
<a href="../../../CODE_OF_CONDUCT/" class="md-nav__link">
Code of Conduct
</a>
</li>
<li class="md-nav__item">
<a href="../../../sre_community/" class="md-nav__link">
SRE Community
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#example-memory-leak" class="md-nav__link">
Example - Memory leak :
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content">
<article class="md-content__inner md-typeset">
<h1>Troubleshooting Example</h1>
<p>In this section we will see an example of an issue and try to troubleshoot it, and at the end a few famous troubleshooting stories are shared, which were shared by LinkedIn engineers earlier.</p>
<h3 id="example-memory-leak">Example - Memory leak :</h3>
<p>Often memory leak issues go unnoticed until the service becomes unresponsive after running for some time (days, week or even month) until service is restarted or bug is fixed, In such cases, service memory usage will reflect in increasing order in the metric graph, something like this graph.</p>
<p><img alt="" src="../images/MemUsageChart.png" /></p>
<p>Memory leak is mismanagement of memory allocations by application, where unneeded memory is not released, over the period of time objects continue to pile up in memory resulting in service crash. Generally such non-released objects get sorted by <a href="https://en.wikipedia.org/wiki/Garbage_collection_(computer_science)">garbage collector</a> automatically, but sometimes due to a bug it fails. Debugging helps in figuring where much of the application storage memory is being applied. Then, you start tracking and filter everything based on usage. In case, you find objects that aren’t in use, but are referenced, you can get rid of them by deleting them to avoid memory leaks. In the case of python applications, it comes with inbuilt features like <a href="https://docs.python.org/3/library/tracemalloc.html">tracemalloc</a>. This module can help pinpoint where an object was allocated first. Almost every language comes with a set of tools/libraries (inbuilt or external) which helps find memory issues. Similarly for Java there is a famous memory leak detection tool called <a href="http://visualvm.java.net/intro.html">Java VisualVM</a>.</p>
<p>Let’s see how a dummy flask based web app with a memory leak bug, with every request its memory usage keeps increasing, and how we can use tracemalloc to capture the leak.</p>
<p>Assumption -: A python virtual environment is created, and flask is installed in it.</p>
<p><strong>A bare minimum flask code with bug, read comments for more info</strong>
<img alt="" src="../images/FlaskCode.png" /></p>
<p><strong>Starting flask app</strong>
<img alt="" src="../images/FlaskStart.png" /></p>
<p><strong>On start, Its memory usage is around 26576 kb, i.e approx 26MB</strong>
<img alt="" src="../images/MemUsage01.png" /></p>
<p><strong>Now with every subsequent GET request, We can notice that process memory usage continues to increase slowly.</strong>
<img alt="" src="../images/MemUsage02.png" /></p>
<p><strong>Now lets try 10000 requests, to see if memory usage increases heavily.</strong>
To hit a high number of requests, we use an Apache benchmarking tool called <a href="https://httpd.apache.org/docs/2.4/programs/ab.html">“ab”</a>. After 10000 hits, we can notice memory usage of flask app is jumped almost 15 times, i.e from initial <strong>26576 KB to 419316 KB, i.e from roughly 26 MB to 419 MB</strong>, That’s a huge jump for such a small webapp.
<img alt="" src="../images/MemUsage03.png" /></p>
<p><strong>Lets try the python <a href="https://docs.python.org/3/library/tracemalloc.html">tracemalloc</a> module to try to understand the application memory allocations.</strong> Tracemalloc takes memory snapshots at a particular point, performing various statistics on the same. </p>
<p>Adding a bare minimum code to our app.py file, no change in fetchuserdata.py file, it will allow us to capture tracemalloc snapshots whenever we will hit /capture uri.
<img alt="" src="../images/Tracemalloc01.png" /></p>
<p><strong>After restart of app.py (flask run)</strong>, we will
- First hit http://127.0.0.1:5000/capture
- Then hit http://127.0.0.1:5000/ 10000 times, for memory leak/s to take place.
- Finally hit http://127.0.0.1:5000/capture again to take a snapshot to know which line has the most allocation.
<img alt="" src="../images/Tracemalloc02.png" /></p>
<p>In the final snapshot, we noticed the exact module and lineno where most allocation happened. I.e fetchuserdata.py, line no 6, after 10000 hits, it is holding 419 MB of memory.
<img alt="" src="../images/Tracemalloc03.png" /></p>
<p><strong>Summary</strong></p>
<p>Above example shows how a bug can lead to memory leak, and how we can use <a href="https://docs.python.org/3/library/tracemalloc.html">tracemalloc</a> to understand where it is. In real world applications are way more complex than the above dummy example, you must understand that using tracemalloc might degrade application performance somebit, due to tracemalloc own overheads. Be mindful about its use in production environments.</p>
<p>If you are interested in digging deeper into Python Object Memory Allocation Internals and debugging memory leak, have a look at an Interesting talk by <a href="https://www.linkedin.com/in/sanketplus/">Sanket Patel</a> in PyCon India 2019, <a href="https://www.youtube.com/watch?v=s9kAghWpzoE">Debug Memory Leak In Python Flask | Python Object Memory Allocation Internals</a></p>
</article>
</div>
</div>
</main>
<footer class="md-footer">
<div class="md-footer-nav">
<nav class="md-footer-nav__inner md-grid" aria-label="Footer">
<a href="../performance-improvements/" class="md-footer-nav__link md-footer-nav__link--prev" rel="prev">
<div class="md-footer-nav__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12z"/></svg>
</div>
<div class="md-footer-nav__title">
<div class="md-ellipsis">
<span class="md-footer-nav__direction">
Previous
</span>
Performance Improvements
</div>
</div>
</a>
<a href="../conclusion/" class="md-footer-nav__link md-footer-nav__link--next" rel="next">
<div class="md-footer-nav__title">
<div class="md-ellipsis">
<span class="md-footer-nav__direction">
Next
</span>
Conclusion
</div>
</div>
<div class="md-footer-nav__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11H4z"/></svg>
</div>
</a>
</nav>
</div>
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-footer-copyright">
<div class="md-footer-copyright__highlight">
Copyright 2020 LinkedIn Corporation. All Rights Reserved. Licensed under the Creative Commons Attribution 4.0 International Public License
</div>
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
<div class="md-footer-social">
<a href="https://github.com/linkedin/school-of-sre" target="_blank" rel="noopener" title="github.com" class="md-footer-social__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
</a>
</div>
</div>
</div>
</footer>
</div>
<script src="../../../assets/javascripts/vendor.18f0862e.min.js"></script>
<script src="../../../assets/javascripts/bundle.994580cf.min.js"></script><script id="__lang" type="application/json">{"clipboard.copy": "Copy to clipboard", "clipboard.copied": "Copied to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.placeholder": "Type to start searching", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.term.missing": "Missing"}</script>
<script>
app = initialize({
base: "../../..",
features: [],
search: Object.assign({
worker: "../../../assets/javascripts/worker/search.9c0e82ba.min.js"
}, typeof search !== "undefined" && search)
})
</script>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink