From 6937cd8694770700da64d7b2b652653d852a6866 Mon Sep 17 00:00:00 2001
From: Kalyanasundaram Somasundaram <ksomasun@ksomasun-mn1.linkedin.biz>
Date: Tue, 3 Nov 2020 19:27:18 +0550
Subject: [PATCH] Deployed 6e8f373 with MkDocs version: 1.0.4

---
 404.html                                  |  42 +++
 git/branches/index.html                   | 383 +++++++++++++++++++
 git/git-basics/index.html                 | 433 ++++++++++++++++++++++
 git/github-hooks/index.html               | 273 ++++++++++++++
 index.html                                |  46 ++-
 python_web/intro/index.html               | 332 +++++++++++++++++
 python_web/python-concepts/index.html     | 370 ++++++++++++++++++
 python_web/python-web-flask/index.html    | 270 ++++++++++++++
 python_web/sre-conclusion/index.html      | 263 +++++++++++++
 python_web/url-shorten-app/index.html     | 332 +++++++++++++++++
 search/search_index.json                  |   2 +-
 sitemap.xml                               |  52 ++-
 sitemap.xml.gz                            | Bin 198 -> 204 bytes
 systems_design/availability/index.html    |  42 +++
 systems_design/conclusion/index.html      |  42 +++
 systems_design/fault-tolerance/index.html |  42 +++
 systems_design/intro/index.html           |  44 ++-
 systems_design/scalability/index.html     |  42 +++
 18 files changed, 3000 insertions(+), 10 deletions(-)
 create mode 100644 git/branches/index.html
 create mode 100644 git/git-basics/index.html
 create mode 100644 git/github-hooks/index.html
 create mode 100644 python_web/intro/index.html
 create mode 100644 python_web/python-concepts/index.html
 create mode 100644 python_web/python-web-flask/index.html
 create mode 100644 python_web/sre-conclusion/index.html
 create mode 100644 python_web/url-shorten-app/index.html
diff --git a/404.html b/404.html
index 3a52f66..098e0e0 100644
--- a/404.html
+++ b/404.html
@@ -49,6 +49,48 @@
                             <li >
                                 <a href="/.">Home</a>
                             </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Git <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="/git/git-basics/">Git Basics</a>
+</li>
+                                    
+<li >
+    <a href="/git/branches/">Working With Branches</a>
+</li>
+                                    
+<li >
+    <a href="/git/github-hooks/">Github and Hooks</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Python and Web <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="/python_web/intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="/python_web/python-concepts/">Some Python Concepts</a>
+</li>
+                                    
+<li >
+    <a href="/python_web/python-web-flask/">Python, Web and Flask</a>
+</li>
+                                    
+<li >
+    <a href="/python_web/url-shorten-app/">The URL Shortening App</a>
+</li>
+                                    
+<li >
+    <a href="/python_web/sre-conclusion/">SRE Aspects of The App and Conclusion</a>
+</li>
+                                </ul>
+                            </li>
                             <li class="dropdown">
                                 <a href="#" class="dropdown-toggle" data-toggle="dropdown">Systems Design <b class="caret"></b></a>
                                 <ul class="dropdown-menu">
diff --git a/git/branches/index.html b/git/branches/index.html
new file mode 100644
index 0000000..87bb864
--- /dev/null
+++ b/git/branches/index.html
@@ -0,0 +1,383 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="utf-8">
+        <meta http-equiv="X-UA-Compatible" content="IE=edge">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        
+        
+        
+        <link rel="shortcut icon" href="../../img/favicon.ico">
+        <title>Working With Branches - school_of_sre</title>
+        <link href="../../css/bootstrap-custom.min.css" rel="stylesheet">
+        <link href="../../css/font-awesome.min.css" rel="stylesheet">
+        <link href="../../css/base.css" rel="stylesheet">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css">
+        <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
+        <!--[if lt IE 9]>
+            <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+            <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
+        <![endif]-->
+
+        <script src="../../js/jquery-1.10.2.min.js" defer></script>
+        <script src="../../js/bootstrap-3.0.3.min.js" defer></script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
+        <script>hljs.initHighlightingOnLoad();</script> 
+    </head>
+
+    <body>
+
+        <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+            <div class="container">
+
+                <!-- Collapsed navigation -->
+                <div class="navbar-header">
+                    <!-- Expander button -->
+                    <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+                        <span class="sr-only">Toggle navigation</span>
+                        <span class="icon-bar"></span>
+                        <span class="icon-bar"></span>
+                        <span class="icon-bar"></span>
+                    </button>
+                    <a class="navbar-brand" href="../..">school_of_sre</a>
+                </div>
+
+                <!-- Expanded navigation -->
+                <div class="navbar-collapse collapse">
+                        <!-- Main navigation -->
+                        <ul class="nav navbar-nav">
+                            <li >
+                                <a href="../..">Home</a>
+                            </li>
+                            <li class="dropdown active">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Git <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../git-basics/">Git Basics</a>
+</li>
+                                    
+<li class="active">
+    <a href="./">Working With Branches</a>
+</li>
+                                    
+<li >
+    <a href="../github-hooks/">Github and Hooks</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Python and Web <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../python_web/intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/python-concepts/">Some Python Concepts</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/python-web-flask/">Python, Web and Flask</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/url-shorten-app/">The URL Shortening App</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/sre-conclusion/">SRE Aspects of The App and Conclusion</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Systems Design <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../systems_design/intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/scalability/">Scalability</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/availability/">Availability</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/fault-tolerance/">Fault Tolerance</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/conclusion/">Conclusion</a>
+</li>
+                                </ul>
+                            </li>
+                        </ul>
+
+                    <ul class="nav navbar-nav navbar-right">
+                        <li>
+                            <a href="#" data-toggle="modal" data-target="#mkdocs_search_modal">
+                                <i class="fa fa-search"></i> Search
+                            </a>
+                        </li>
+                            <li >
+                                <a rel="next" href="../git-basics/">
+                                    <i class="fa fa-arrow-left"></i> Previous
+                                </a>
+                            </li>
+                            <li >
+                                <a rel="prev" href="../github-hooks/">
+                                    Next <i class="fa fa-arrow-right"></i>
+                                </a>
+                            </li>
+                    </ul>
+                </div>
+            </div>
+        </div>
+
+        <div class="container">
+                <div class="col-md-3"><div class="bs-sidebar hidden-print affix well" role="complementary">
+    <ul class="nav bs-sidenav">
+        <li class="main active"><a href="#working-with-branches">Working With Branches</a></li>
+            <li><a href="#merges">Merges</a></li>
+    </ul>
+</div></div>
+                <div class="col-md-9" role="main">
+
+<h1 id="working-with-branches">Working With Branches</h1>
+<p>Coming back to our local repo which has two commits. So far, what we have is a single line of history. Commits are chained in a single line. But sometimes you may have a need to work on two different features in parallel in the same repo. Now one option here could be making a new folder/repo with the same code and use that for another feature development. But there's a better way. Use <em>branches.</em> Since git follows tree like structure for commits, we can use branches to work on different sets of features. From a commit, two or more branches can be created and branches can also be merged.</p>
+<p>Using branches, there can exist multiple lines of histories and we can checkout to any of them and work on it. Checking out, as we discussed earlier, would simply mean replacing contents of the directory (repo) with contents snapshot at the checked out version.</p>
+<p>Let's create a branch and see how it looks like:</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ git branch b1
+spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph
+* 7f3b00e (HEAD -&gt; master, b1) adding file 2
+* df2fb7a adding file 1
+</code></pre>
+
+<p>We create a branch called <code>b1</code>. Git log tells us that b1 also points to the last commit (7f3b00e) but the <code>HEAD</code> is still pointing to master. If you remember, HEAD points to the commit/reference wherever you are checkout to. So if we checkout to <code>b1</code>, HEAD should point to that. Let's confirm:</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ git checkout b1
+Switched to branch 'b1'
+spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph
+* 7f3b00e (HEAD -&gt; b1, master) adding file 2
+* df2fb7a adding file 1
+</code></pre>
+
+<p><code>b1</code> still points to the same commit but HEAD now points to <code>b1</code>. Since we create a branch at commit <code>7f3b00e</code>, there will be two lines of histories starting this commit. Depending on which branch you are checked out on, the line of history will progress.</p>
+<p>At this moment, we are checked out on branch <code>b1</code>, so making a new commit will advance branch reference <code>b1</code> to that commit and current <code>b1</code> commit will become its parent. Let's do that.</p>
+<pre><code class="bash"># Creating a file and making a commit
+spatel1-mn1:school-of-sre spatel1$ echo &quot;I am a file in b1 branch&quot; &gt; b1.txt
+spatel1-mn1:school-of-sre spatel1$ git add b1.txt
+spatel1-mn1:school-of-sre spatel1$ git commit -m &quot;adding b1 file&quot;
+[b1 872a38f] adding b1 file
+1 file changed, 1 insertion(+)
+create mode 100644 b1.txt
+
+# The new line of history
+spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph
+* 872a38f (HEAD -&gt; b1) adding b1 file
+* 7f3b00e (master) adding file 2
+* df2fb7a adding file 1
+spatel1-mn1:school-of-sre spatel1$
+</code></pre>
+
+<p>Do note that master is still pointing to the old commit it was pointing to. We can now checkout to master branch and make commits there. This will result in another line of history starting from commit 7f3b00e.</p>
+<pre><code class="bash"># checkout to master branch
+spatel1-mn1:school-of-sre spatel1$ git checkout master
+Switched to branch 'master'
+
+# Creating a new commit on master branch
+spatel1-mn1:school-of-sre spatel1$ echo &quot;new file in master branch&quot; &gt; master.txt
+spatel1-mn1:school-of-sre spatel1$ git add master.txt
+spatel1-mn1:school-of-sre spatel1$ git commit -m &quot;adding master.txt file&quot;
+[master 60dc441] adding master.txt file
+1 file changed, 1 insertion(+)
+create mode 100644 master.txt
+
+# The history line
+spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph
+* 60dc441 (HEAD -&gt; master) adding master.txt file
+* 7f3b00e adding file 2
+* df2fb7a adding file 1
+</code></pre>
+
+<p>Notice how branch b1 is not visible here since we are checkout on master. Let's try to visualize both to get the whole picture:</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all
+* 60dc441 (HEAD -&gt; master) adding master.txt file
+| * 872a38f (b1) adding b1 file
+|/ 
+* 7f3b00e adding file 2
+* df2fb7a adding file 1
+</code></pre>
+
+<p>Above tree structure should make things clear. Notice a clear branch/fork on commit 7f3b00e. This is how we create branches. Now they both are two separate lines of history on which feature development can be done independently.</p>
+<p><strong>To reiterate, internally, git is just a tree of commits. Branch names (human readable) are pointers to those commits in the tree. We use various git commands to work with the tree structure and references. Git accordingly modifies contents of our repo.</strong></p>
+<h2 id="merges">Merges</h2>
+<p>Now say the feature you were working on branch <code>b1</code> is complete. And you need to merge it on master branch, where all the final version of code goes. So first you will checkout to branch master and then you will pull the latest code from upstream (eg: GitHub). Then you need to merge your code from <code>b1</code> into master. And there could be two ways this can be done.</p>
+<p>Here is the current history:</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all
+* 60dc441 (HEAD -&gt; master) adding master.txt file
+| * 872a38f (b1) adding b1 file
+|/ 
+* 7f3b00e adding file 2
+* df2fb7a adding file 1
+</code></pre>
+
+<p><strong>Option 1: Directly merge the branch.</strong> Merging the branch b1 into master will result in a new merge commit which will merge changes from two different lines of history and create a new commit of the result.</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ git merge b1
+Merge made by the 'recursive' strategy.
+b1.txt | 1 +
+1 file changed, 1 insertion(+)
+create mode 100644 b1.txt
+spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all
+*   8fc28f9 (HEAD -&gt; master) Merge branch 'b1'
+|\
+| * 872a38f (b1) adding b1 file
+* | 60dc441 adding master.txt file
+|/ 
+* 7f3b00e adding file 2
+* df2fb7a adding file 1
+</code></pre>
+
+<p>You can see a new merge commit created (8fc28f9). You will be prompted for the commit message. If there are a lot of branches in the repo, this result will end-up with a lot of merge commits. Which looks ugly compared to a single line of history of development. So let's look at an alternative approach</p>
+<p>First let's <a href="https://git-scm.com/docs/git-reset">reset</a> our last merge and go to the previous state.</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ git reset --hard 60dc441
+HEAD is now at 60dc441 adding master.txt file
+spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all
+* 60dc441 (HEAD -&gt; master) adding master.txt file
+| * 872a38f (b1) adding b1 file
+|/ 
+* 7f3b00e adding file 2
+* df2fb7a adding file 1
+</code></pre>
+
+<p><strong>Option 2: Rebase.</strong> Now, instead of merging two branches which has a similar base (commit: 7f3b00e), let us rebase branch b1 on to current master. <strong>What this means is take branch <code>b1</code> (from commit 7f3b00e to commit 872a38f) and rebase (put them on top of) master (60dc441).</strong></p>
+<pre><code class="bash"># Switch to b1
+spatel1-mn1:school-of-sre spatel1$ git checkout b1
+Switched to branch 'b1'
+
+# Rebase (b1 which is current branch) on master
+spatel1-mn1:school-of-sre spatel1$ git rebase master
+First, rewinding head to replay your work on top of it...
+Applying: adding b1 file
+
+# The result
+spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all
+* 5372c8f (HEAD -&gt; b1) adding b1 file
+* 60dc441 (master) adding master.txt file
+* 7f3b00e adding file 2
+* df2fb7a adding file 1
+</code></pre>
+
+<p>You can see <code>b1</code> which had 1 commit. That commit's parent was <code>7f3b00e</code>. But since we rebase it on master (<code>60dc441</code>). That becomes the parent now. As a side effect, you also see it has become a single line of history. Now if we were to merge <code>b1</code> into <code>master</code>, it would simply mean change <code>master</code> to point to <code>5372c8f</code> which is <code>b1</code>. Let's try it:</p>
+<pre><code class="bash"># checkout to master since we want to merge code into master
+spatel1-mn1:school-of-sre spatel1$ git checkout master
+Switched to branch 'master'
+
+# the current history, where b1 is based on master
+spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all
+* 5372c8f (b1) adding b1 file
+* 60dc441 (HEAD -&gt; master) adding master.txt file
+* 7f3b00e adding file 2
+* df2fb7a adding file 1
+
+
+# Performing the merge, notice the &quot;fast-forward&quot; message
+spatel1-mn1:school-of-sre spatel1$ git merge b1
+Updating 60dc441..5372c8f
+Fast-forward
+b1.txt | 1 +
+1 file changed, 1 insertion(+)
+create mode 100644 b1.txt
+
+# The Result
+spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all
+* 5372c8f (HEAD -&gt; master, b1) adding b1 file
+* 60dc441 adding master.txt file
+* 7f3b00e adding file 2
+* df2fb7a adding file 1
+</code></pre>
+
+<p>Now you see both <code>b1</code> and <code>master</code> are pointing to the same commit. Your code has been merged to the master branch and it can be pushed. Also we have clean line of history! :D</p></div>
+        </div>
+
+        <footer class="col-md-12">
+            <hr>
+            <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
+        </footer>
+        <script>
+            var base_url = "../..",
+                shortcuts = {"search": 83, "next": 78, "help": 191, "previous": 80};
+        </script>
+        <script src="../../js/base.js" defer></script>
+        <script src="../../search/main.js" defer></script>
+
+        <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="Search Modal" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+                <h4 class="modal-title" id="exampleModalLabel">Search</h4>
+            </div>
+            <div class="modal-body">
+                <p>
+                    From here you can search these documents. Enter
+                    your search terms below.
+                </p>
+                <form role="form">
+                    <div class="form-group">
+                        <input type="text" class="form-control" placeholder="Search..." id="mkdocs-search-query" title="Type search term here">
+                    </div>
+                </form>
+                <div id="mkdocs-search-results"></div>
+            </div>
+            <div class="modal-footer">
+            </div>
+        </div>
+    </div>
+</div><div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="Keyboard Shortcuts Modal" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+                <h4 class="modal-title" id="exampleModalLabel">Keyboard Shortcuts</h4>
+            </div>
+            <div class="modal-body">
+              <table class="table">
+                <thead>
+                  <tr>
+                    <th style="width: 20%;">Keys</th>
+                    <th>Action</th>
+                  </tr>
+                </thead>
+                <tbody>
+                  <tr>
+                    <td class="help shortcut"><kbd>?</kbd></td>
+                    <td>Open this help</td>
+                  </tr>
+                  <tr>
+                    <td class="next shortcut"><kbd>n</kbd></td>
+                    <td>Next page</td>
+                  </tr>
+                  <tr>
+                    <td class="prev shortcut"><kbd>p</kbd></td>
+                    <td>Previous page</td>
+                  </tr>
+                  <tr>
+                    <td class="search shortcut"><kbd>s</kbd></td>
+                    <td>Search</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+            <div class="modal-footer">
+            </div>
+        </div>
+    </div>
+</div>
+
+    </body>
+</html>
diff --git a/git/git-basics/index.html b/git/git-basics/index.html
new file mode 100644
index 0000000..9ae10e5
--- /dev/null
+++ b/git/git-basics/index.html
@@ -0,0 +1,433 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="utf-8">
+        <meta http-equiv="X-UA-Compatible" content="IE=edge">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        
+        
+        
+        <link rel="shortcut icon" href="../../img/favicon.ico">
+        <title>Git Basics - school_of_sre</title>
+        <link href="../../css/bootstrap-custom.min.css" rel="stylesheet">
+        <link href="../../css/font-awesome.min.css" rel="stylesheet">
+        <link href="../../css/base.css" rel="stylesheet">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css">
+        <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
+        <!--[if lt IE 9]>
+            <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+            <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
+        <![endif]-->
+
+        <script src="../../js/jquery-1.10.2.min.js" defer></script>
+        <script src="../../js/bootstrap-3.0.3.min.js" defer></script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
+        <script>hljs.initHighlightingOnLoad();</script> 
+    </head>
+
+    <body>
+
+        <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+            <div class="container">
+
+                <!-- Collapsed navigation -->
+                <div class="navbar-header">
+                    <!-- Expander button -->
+                    <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+                        <span class="sr-only">Toggle navigation</span>
+                        <span class="icon-bar"></span>
+                        <span class="icon-bar"></span>
+                        <span class="icon-bar"></span>
+                    </button>
+                    <a class="navbar-brand" href="../..">school_of_sre</a>
+                </div>
+
+                <!-- Expanded navigation -->
+                <div class="navbar-collapse collapse">
+                        <!-- Main navigation -->
+                        <ul class="nav navbar-nav">
+                            <li >
+                                <a href="../..">Home</a>
+                            </li>
+                            <li class="dropdown active">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Git <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li class="active">
+    <a href="./">Git Basics</a>
+</li>
+                                    
+<li >
+    <a href="../branches/">Working With Branches</a>
+</li>
+                                    
+<li >
+    <a href="../github-hooks/">Github and Hooks</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Python and Web <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../python_web/intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/python-concepts/">Some Python Concepts</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/python-web-flask/">Python, Web and Flask</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/url-shorten-app/">The URL Shortening App</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/sre-conclusion/">SRE Aspects of The App and Conclusion</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Systems Design <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../systems_design/intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/scalability/">Scalability</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/availability/">Availability</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/fault-tolerance/">Fault Tolerance</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/conclusion/">Conclusion</a>
+</li>
+                                </ul>
+                            </li>
+                        </ul>
+
+                    <ul class="nav navbar-nav navbar-right">
+                        <li>
+                            <a href="#" data-toggle="modal" data-target="#mkdocs_search_modal">
+                                <i class="fa fa-search"></i> Search
+                            </a>
+                        </li>
+                            <li >
+                                <a rel="next" href="../..">
+                                    <i class="fa fa-arrow-left"></i> Previous
+                                </a>
+                            </li>
+                            <li >
+                                <a rel="prev" href="../branches/">
+                                    Next <i class="fa fa-arrow-right"></i>
+                                </a>
+                            </li>
+                    </ul>
+                </div>
+            </div>
+        </div>
+
+        <div class="container">
+                <div class="col-md-3"><div class="bs-sidebar hidden-print affix well" role="complementary">
+    <ul class="nav bs-sidenav">
+        <li class="main active"><a href="#school-of-sre-git">School Of SRE: Git</a></li>
+            <li><a href="#pre-reads">Pre - Reads</a></li>
+            <li><a href="#what-to-expect-from-this-training">What to expect from this training</a></li>
+            <li><a href="#what-is-not-covered-under-this-training">What is not covered under this training</a></li>
+            <li><a href="#training-content">Training Content</a></li>
+            <li><a href="#git-basics">Git Basics</a></li>
+    </ul>
+</div></div>
+                <div class="col-md-9" role="main">
+
+<h1 id="school-of-sre-git">School Of SRE: Git</h1>
+<h2 id="pre-reads">Pre - Reads</h2>
+<ol>
+<li>Have Git installed <a href="https://git-scm.com/downloads">https://git-scm.com/downloads</a></li>
+<li>Have taken any git high level tutorial or following LinkedIn learning courses<ul>
+<li><a href="https://www.linkedin.com/learning/git-essential-training-the-basics/">https://www.linkedin.com/learning/git-essential-training-the-basics/</a></li>
+<li><a href="https://www.linkedin.com/learning/git-branches-merges-and-remotes/">https://www.linkedin.com/learning/git-branches-merges-and-remotes/</a></li>
+<li><a href="https://git-scm.com/doc">The Official Git Docs</a></li>
+</ul>
+</li>
+</ol>
+<h2 id="what-to-expect-from-this-training">What to expect from this training</h2>
+<p>As an engineer in the field of computer science, having knowledge of version control tools becomes almost a requirement. While there are a lot of version control tools that exist today, Git perhaps is the most used one and this course we will be working with Git. While this course does not start with Git 101 and expects basic knowledge of git as a prerequisite, it will reintroduce the git concepts known by you with details covering what is happening under the hood as you execute various git commands. So that next time you run a git command, you will be able to press enter more confidently!</p>
+<h2 id="what-is-not-covered-under-this-training">What is not covered under this training</h2>
+<p>Advanced usage and specifics of internal implementation details of Git.</p>
+<h2 id="training-content">Training Content</h2>
+<h3 id="table-of-contents">Table of Contents</h3>
+<ol>
+<li>Git Basics</li>
+<li>Working with Branches</li>
+<li>Git with Github</li>
+<li>Hooks</li>
+</ol>
+<h2 id="git-basics">Git Basics</h2>
+<p>Though you might be aware already, let's revisit why we need a version control system. As the project grows and multiple developers start working on it, an efficient method for collaboration is warranted. Git helps the team collaborate easily and also maintains history of the changes happened with the codebase.</p>
+<h3 id="creating-a-git-repo">Creating a Git Repo</h3>
+<p>Any folder can be converted into a git repository. After executing the following command, we will see a <code>.git</code> folder within the folder, which makes our folder a git repository. <strong>All the magic that git does, <code>.git</code> folder is the enabler for the same.</strong></p>
+<pre><code class="bash"># creating an empty folder and changing current dir to it
+spatel1-mn1:~ spatel1$ cd /tmp
+spatel1-mn1:tmp spatel1$ mkdir school-of-sre
+spatel1-mn1:tmp spatel1$ cd school-of-sre/
+
+# initialize a git repo
+spatel1-mn1:school-of-sre spatel1$ git init
+Initialized empty Git repository in /private/tmp/school-of-sre/.git/
+</code></pre>
+
+<p>As the output says, an empty git repo has been initialized in our folder. Let's take a look at what is there.</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ ls .git/
+HEAD        config      description hooks       info        objects     refs
+</code></pre>
+
+<p>There are a bunch of folders and files in the <code>.git</code> folder. As I said, all these enables git to do its magic. We will look into some of these folders and files. But for now, what we have is an empty git repository.</p>
+<h3 id="tracking-a-file">Tracking a File</h3>
+<p>Now as you might already know, let us create a new file in our repo (we will refer to the folder as <em>repo</em> now.) And see git status</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ echo &quot;I am file 1&quot; &gt; file1.txt
+spatel1-mn1:school-of-sre spatel1$ git status
+On branch master
+
+No commits yet
+
+Untracked files:
+ (use &quot;git add &lt;file&gt;...&quot; to include in what will be committed)
+
+       file1.txt
+
+nothing added to commit but untracked files present (use &quot;git add&quot; to track)
+</code></pre>
+
+<p>The current git status says <code>No commits yet</code> and there is one untracked file. Since we just created the file, git is not tracking that file. We explicitly need to ask git to track files and folders. (also checkout <a href="https://git-scm.com/docs/gitignore">gitignore</a>) And how we do that is via <code>git add</code> command as suggested in the above output. Then we go ahead and create a  commit.</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ git add file1.txt
+spatel1-mn1:school-of-sre spatel1$ git status
+On branch master
+
+No commits yet
+
+Changes to be committed:
+ (use &quot;git rm --cached &lt;file&gt;...&quot; to unstage)
+
+       new file:   file1.txt
+
+spatel1-mn1:school-of-sre spatel1$ git commit -m &quot;adding file 1&quot;
+[master (root-commit) df2fb7a] adding file 1
+1 file changed, 1 insertion(+)
+create mode 100644 file1.txt
+</code></pre>
+
+<p>Notice how after adding the file, git status says <code>Changes to be commited:</code>. What it means is whatever is listed there, will be included in the next commit. Then we go ahead and create a commit, with an attached messaged via <code>-m</code>.</p>
+<h3 id="more-about-a-commit">More About a Commit</h3>
+<p>Commit is a snapshot of the repo. Whenever a commit is made, a snapshot of the current state of repo (the folder) is taken and saved. Each commit has a unique ID. (<code>df2fb7a</code> for the commit we made in the previous step). As we keep adding/changing more and more contents and keep making commits, all those snapshots are stored by git. Again, all this magic happens inside the <code>.git</code> folder. This is where all this snapshot or versions are stored. <em>In an efficient manner.</em></p>
+<h3 id="adding-more-changes">Adding More Changes</h3>
+<p>Let us create one more file and commit the change. It would look the same as the previous commit we made.</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ echo &quot;I am file 2&quot; &gt; file2.txt
+spatel1-mn1:school-of-sre spatel1$ git add file2.txt
+spatel1-mn1:school-of-sre spatel1$ git commit -m &quot;adding file 2&quot;
+[master 7f3b00e] adding file 2
+1 file changed, 1 insertion(+)
+create mode 100644 file2.txt
+</code></pre>
+
+<p>A new commit with ID <code>7f3b00e</code> has been created. You can issue <code>git status</code> at any time to see the state of the repository.</p>
+<pre><code>   **IMPORTANT: Note that commit IDs are long string (SHA) but we can refer to a commit by its initial few (8 or more) characters too. We will interchangeably using shorter and longer commit IDs.**
+</code></pre>
+<p>Now that we have two commits, let's visualize them:</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph
+* 7f3b00e (HEAD -&gt; master) adding file 2
+* df2fb7a adding file 1
+</code></pre>
+
+<p><code>git log</code>, as the name suggests, prints the log of all the git commits. Here you see two additional arguments, <code>--oneline</code> prints the shorter version of the log, ie: the commit message only and not the person who made the commit and when. <code>--graph</code> prints it in graph format.</p>
+<p><strong>Now at this moment the commits might look like just one in each line but all commits are stored as a tree like data structure internally by git. That means there can be two or more children commits of a given commit. And not just a single line of commits. We will look more into this part when we get to the Branches section. For now this is our commit history:</strong></p>
+<pre><code class="bash">   df2fb7a ===&gt; 7f3b00e
+</code></pre>
+
+<h3 id="are-commits-really-linked">Are commits really linked?</h3>
+<p>As I just said, the two commits we just made are linked via tree like data structure and we saw how they are linked. But let's actually verify it. Everything in git is an object. Newly created files are stored as an object. Changes to file are stored as an objects and even commits are objects. To view contents of an object we can use the following command with the object's ID. We will take a look at content of the contents of the second commit</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ git cat-file -p 7f3b00e
+tree ebf3af44d253e5328340026e45a9fa9ae3ea1982
+parent df2fb7a61f5d40c1191e0fdeb0fc5d6e7969685a
+author Sanket Patel &lt;spatel1@linkedin.com&gt; 1603273316 -0700
+committer Sanket Patel &lt;spatel1@linkedin.com&gt; 1603273316 -0700
+
+adding file 2
+</code></pre>
+
+<p>Take a note of <code>parent</code> attribute in the above output. It points to the commit id of the first commit we made. So this proves that they are linked! Additionally you can see the second commit's message in this object. As I said all this magic is enabled by <code>.git</code> folder and the object to which we are looking at also is in that folder.</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ ls .git/objects/7f/3b00eaa957815884198e2fdfec29361108d6a9
+.git/objects/7f/3b00eaa957815884198e2fdfec29361108d6a9
+</code></pre>
+
+<p>It is stored in <code>.git/objects/</code> folder. All the files and changes to them as well are stored in this folder.</p>
+<h3 id="the-version-control-part-of-git">The Version Control part of Git</h3>
+<p>We already can see two commits (versions) in our git log. One thing a version control tool gives you is ability to browse back and forth in history. For example: some of your users are running an old version of code and they are reporting an issue. In order to debug the issue, you need access to the old code. The one in your current repo is the latest code. In this example, you are working on the second commit (7f3b00e) and someone reported an issue with the code snapshot at commit (df2fb7a). This is how you would get access to the code at any older commit</p>
+<pre><code class="bash"># Current contents, two files present
+patel1-mn1:school-of-sre spatel1$ ls
+file1.txt file2.txt
+
+# checking out to (an older) commit
+spatel1-mn1:school-of-sre spatel1$ git checkout df2fb7a
+Note: checking out 'df2fb7a'.
+
+You are in 'detached HEAD' state. You can look around, make experimental
+changes and commit them, and you can discard any commits you make in this
+state without impacting any branches by performing another checkout.
+
+If you want to create a new branch to retain commits you create, you may
+do so (now or later) by using -b with the checkout command again. Example:
+
+ git checkout -b &lt;new-branch-name&gt;
+
+HEAD is now at df2fb7a adding file 1
+
+# checking contents, can verify it has old contents
+spatel1-mn1:school-of-sre spatel1$ ls
+file1.txt
+</code></pre>
+
+<p>So this is how we would get access to old versions/snapshots. All we need is a <em>reference</em> to that snapshot. Upon executing <code>git checkout ...</code>, what git does for you is use the <code>.git</code> folder, see what was the state of things (files and folders) at that version/reference and replace the contents of current directory with those contents. The then-existing content will no longer be present in the local dir (repo) but we can and will still get access to them because they are tracked via git commit and <code>.git</code> folder has them stored/tracked.</p>
+<h3 id="reference">Reference</h3>
+<p>I mention in the previous section that we need a <em>reference</em> to the version. By default, git repo is made of tree of commits. And each commit has a unique IDs. But the unique ID is not the only thing we can reference commits via. There are multiple ways to reference commits. For example: <code>HEAD</code> is a reference to current commit. <em>Whatever commit your repo is checked out at, <code>HEAD</code> will point to that.</em> <code>HEAD~1</code> is reference to previous commit. So while checking out previous version in section above, we could have done <code>git checkout HEAD~1</code>.</p>
+<p>Similarly, master is also a reference (to a branch). Since git uses tree like structure to store commits, there of course will be branches. And the default branch is called <code>master</code>. Master (or any branch reference) will point to the latest commit in the branch. Even though we have checked out to the previous commit in out repo, <code>master</code> still points to the latest commit. And we can get back to the latest version by checkout at <code>master</code> reference</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ git checkout master
+Previous HEAD position was df2fb7a adding file 1
+Switched to branch 'master'
+
+# now we will see latest code, with two files
+spatel1-mn1:school-of-sre spatel1$ ls
+file1.txt file2.txt
+</code></pre>
+
+<p>Note, instead of <code>master</code> in above command, we could have used commit's ID as well.</p>
+<h3 id="references-and-the-magic">References and The Magic</h3>
+<p>Let's look at the state of things. Two commits, <code>master</code> and <code>HEAD</code> references are pointing to the latest commit</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph
+* 7f3b00e (HEAD -&gt; master) adding file 2
+* df2fb7a adding file 1
+</code></pre>
+
+<p>The magic? Let's examine these files:</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ cat .git/refs/heads/master
+7f3b00eaa957815884198e2fdfec29361108d6a9
+</code></pre>
+
+<p>Viola! Where master is pointing to is stored in a file. <strong>Whenever git needs to know where master reference is pointing to, or if git needs to update where master points, it just needs to update the file above.</strong> So when you create a new commit, a new commit is created on top of the current commit and the master file is updated with the new commit's ID.</p>
+<p>Similary, for <code>HEAD</code> reference:</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ cat .git/HEAD
+ref: refs/heads/master
+</code></pre>
+
+<p>We can see <code>HEAD</code> is pointing to a reference called <code>refs/heads/master</code>. So <code>HEAD</code> will point where ever the <code>master</code> points.</p>
+<h3 id="little-adventure">Little Adventure</h3>
+<p>We discussed how git will update the files as we execute commands. But let's try to do it ourselves, by hand, and see what happens.</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph
+* 7f3b00e (HEAD -&gt; master) adding file 2
+* df2fb7a adding file 1
+</code></pre>
+
+<p>Now let's change master to point to the previous/first commit.</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ echo df2fb7a61f5d40c1191e0fdeb0fc5d6e7969685a &gt; .git/refs/heads/master
+spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph
+* df2fb7a (HEAD -&gt; master) adding file 1
+
+# RESETTING TO ORIGINAL
+spatel1-mn1:school-of-sre spatel1$ echo 7f3b00eaa957815884198e2fdfec29361108d6a9 &gt; .git/refs/heads/master
+spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph
+* 7f3b00e (HEAD -&gt; master) adding file 2
+* df2fb7a adding file 1
+</code></pre>
+
+<p>We just edited the <code>master</code> reference file and now we can see only the first commit in git log. Undoing the change to the file brings the state back to original. Not so much of magic, isn't it?</p></div>
+        </div>
+
+        <footer class="col-md-12">
+            <hr>
+            <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
+        </footer>
+        <script>
+            var base_url = "../..",
+                shortcuts = {"search": 83, "next": 78, "help": 191, "previous": 80};
+        </script>
+        <script src="../../js/base.js" defer></script>
+        <script src="../../search/main.js" defer></script>
+
+        <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="Search Modal" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+                <h4 class="modal-title" id="exampleModalLabel">Search</h4>
+            </div>
+            <div class="modal-body">
+                <p>
+                    From here you can search these documents. Enter
+                    your search terms below.
+                </p>
+                <form role="form">
+                    <div class="form-group">
+                        <input type="text" class="form-control" placeholder="Search..." id="mkdocs-search-query" title="Type search term here">
+                    </div>
+                </form>
+                <div id="mkdocs-search-results"></div>
+            </div>
+            <div class="modal-footer">
+            </div>
+        </div>
+    </div>
+</div><div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="Keyboard Shortcuts Modal" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+                <h4 class="modal-title" id="exampleModalLabel">Keyboard Shortcuts</h4>
+            </div>
+            <div class="modal-body">
+              <table class="table">
+                <thead>
+                  <tr>
+                    <th style="width: 20%;">Keys</th>
+                    <th>Action</th>
+                  </tr>
+                </thead>
+                <tbody>
+                  <tr>
+                    <td class="help shortcut"><kbd>?</kbd></td>
+                    <td>Open this help</td>
+                  </tr>
+                  <tr>
+                    <td class="next shortcut"><kbd>n</kbd></td>
+                    <td>Next page</td>
+                  </tr>
+                  <tr>
+                    <td class="prev shortcut"><kbd>p</kbd></td>
+                    <td>Previous page</td>
+                  </tr>
+                  <tr>
+                    <td class="search shortcut"><kbd>s</kbd></td>
+                    <td>Search</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+            <div class="modal-footer">
+            </div>
+        </div>
+    </div>
+</div>
+
+    </body>
+</html>
diff --git a/git/github-hooks/index.html b/git/github-hooks/index.html
new file mode 100644
index 0000000..13c6e36
--- /dev/null
+++ b/git/github-hooks/index.html
@@ -0,0 +1,273 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="utf-8">
+        <meta http-equiv="X-UA-Compatible" content="IE=edge">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        
+        
+        
+        <link rel="shortcut icon" href="../../img/favicon.ico">
+        <title>Github and Hooks - school_of_sre</title>
+        <link href="../../css/bootstrap-custom.min.css" rel="stylesheet">
+        <link href="../../css/font-awesome.min.css" rel="stylesheet">
+        <link href="../../css/base.css" rel="stylesheet">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css">
+        <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
+        <!--[if lt IE 9]>
+            <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+            <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
+        <![endif]-->
+
+        <script src="../../js/jquery-1.10.2.min.js" defer></script>
+        <script src="../../js/bootstrap-3.0.3.min.js" defer></script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
+        <script>hljs.initHighlightingOnLoad();</script> 
+    </head>
+
+    <body>
+
+        <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+            <div class="container">
+
+                <!-- Collapsed navigation -->
+                <div class="navbar-header">
+                    <!-- Expander button -->
+                    <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+                        <span class="sr-only">Toggle navigation</span>
+                        <span class="icon-bar"></span>
+                        <span class="icon-bar"></span>
+                        <span class="icon-bar"></span>
+                    </button>
+                    <a class="navbar-brand" href="../..">school_of_sre</a>
+                </div>
+
+                <!-- Expanded navigation -->
+                <div class="navbar-collapse collapse">
+                        <!-- Main navigation -->
+                        <ul class="nav navbar-nav">
+                            <li >
+                                <a href="../..">Home</a>
+                            </li>
+                            <li class="dropdown active">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Git <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../git-basics/">Git Basics</a>
+</li>
+                                    
+<li >
+    <a href="../branches/">Working With Branches</a>
+</li>
+                                    
+<li class="active">
+    <a href="./">Github and Hooks</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Python and Web <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../python_web/intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/python-concepts/">Some Python Concepts</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/python-web-flask/">Python, Web and Flask</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/url-shorten-app/">The URL Shortening App</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/sre-conclusion/">SRE Aspects of The App and Conclusion</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Systems Design <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../systems_design/intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/scalability/">Scalability</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/availability/">Availability</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/fault-tolerance/">Fault Tolerance</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/conclusion/">Conclusion</a>
+</li>
+                                </ul>
+                            </li>
+                        </ul>
+
+                    <ul class="nav navbar-nav navbar-right">
+                        <li>
+                            <a href="#" data-toggle="modal" data-target="#mkdocs_search_modal">
+                                <i class="fa fa-search"></i> Search
+                            </a>
+                        </li>
+                            <li >
+                                <a rel="next" href="../branches/">
+                                    <i class="fa fa-arrow-left"></i> Previous
+                                </a>
+                            </li>
+                            <li >
+                                <a rel="prev" href="../../python_web/intro/">
+                                    Next <i class="fa fa-arrow-right"></i>
+                                </a>
+                            </li>
+                    </ul>
+                </div>
+            </div>
+        </div>
+
+        <div class="container">
+                <div class="col-md-3"><div class="bs-sidebar hidden-print affix well" role="complementary">
+    <ul class="nav bs-sidenav">
+        <li class="main active"><a href="#git-with-github">Git with Github</a></li>
+        <li class="main "><a href="#hooks">Hooks</a></li>
+        <li class="main "><a href="#what-next-from-here">What next from here?</a></li>
+    </ul>
+</div></div>
+                <div class="col-md-9" role="main">
+
+<h2 id="git-with-github">Git with Github</h2>
+<p>Till now all the operations we did were in our local repo while git also helps us in a collaborative environment. GitHub is one place on the internet where you can centrally host your git repos and collaborate with other developers.</p>
+<p>Most of the workflow will remain the same as we discussed, with addition of couple of things:</p>
+<ol>
+<li>Pull: to pull latest changes from github (the central) repo</li>
+<li>Push: to push your changes to github repo so that it's available to all people</li>
+</ol>
+<p>GitHub has written nice guides and tutorials about this and you can refer them here:</p>
+<ul>
+<li><a href="https://guides.github.com/activities/hello-world/">GitHub Hello World</a></li>
+<li><a href="https://guides.github.com/introduction/git-handbook/">Git Handbook</a></li>
+</ul>
+<h2 id="hooks">Hooks</h2>
+<p>Git has another nice feature called hooks. Hooks are basically scripts which will be called when a certain event happens. Here is where hooks are located:</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ ls .git/hooks/
+applypatch-msg.sample     fsmonitor-watchman.sample pre-applypatch.sample     pre-push.sample           pre-receive.sample        update.sample
+commit-msg.sample         post-update.sample        pre-commit.sample         pre-rebase.sample         prepare-commit-msg.sample
+</code></pre>
+
+<p>Names are self explanatory. These hooks are useful when you want to do certain things when a certain event happens. Ie: if you want to run tests before pushing code, you would want to setup <code>pre-push</code> hooks. Let's try to create a pre commit hook.</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ echo &quot;echo this is from pre commit hook&quot; &gt; .git/hooks/pre-commit
+spatel1-mn1:school-of-sre spatel1$ chmod +x .git/hooks/pre-commit
+</code></pre>
+
+<p>We basically create a file called <code>pre-commit</code> in hooks folder and make it executable. Now if we make a commit, we should see the message getting printed.</p>
+<pre><code class="bash">spatel1-mn1:school-of-sre spatel1$ echo &quot;sample file&quot; &gt; sample.txt
+spatel1-mn1:school-of-sre spatel1$ git add sample.txt
+spatel1-mn1:school-of-sre spatel1$ git commit -m &quot;adding sample file&quot;
+this is from pre commit hook     # &lt;===== THE MESSAGE FROM HOOK EXECUTION
+[master 9894e05] adding sample file
+1 file changed, 1 insertion(+)
+create mode 100644 sample.txt
+</code></pre>
+
+<h2 id="what-next-from-here">What next from here?</h2>
+<p>There are a lot of git commands and features which we have not explored here. But with the base built-up, be sure to explore concepts like</p>
+<ul>
+<li>Cherrypick</li>
+<li>Squash</li>
+<li>Amend</li>
+<li>Stash</li>
+<li>Reset</li>
+</ul></div>
+        </div>
+
+        <footer class="col-md-12">
+            <hr>
+            <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
+        </footer>
+        <script>
+            var base_url = "../..",
+                shortcuts = {"search": 83, "next": 78, "help": 191, "previous": 80};
+        </script>
+        <script src="../../js/base.js" defer></script>
+        <script src="../../search/main.js" defer></script>
+
+        <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="Search Modal" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+                <h4 class="modal-title" id="exampleModalLabel">Search</h4>
+            </div>
+            <div class="modal-body">
+                <p>
+                    From here you can search these documents. Enter
+                    your search terms below.
+                </p>
+                <form role="form">
+                    <div class="form-group">
+                        <input type="text" class="form-control" placeholder="Search..." id="mkdocs-search-query" title="Type search term here">
+                    </div>
+                </form>
+                <div id="mkdocs-search-results"></div>
+            </div>
+            <div class="modal-footer">
+            </div>
+        </div>
+    </div>
+</div><div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="Keyboard Shortcuts Modal" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+                <h4 class="modal-title" id="exampleModalLabel">Keyboard Shortcuts</h4>
+            </div>
+            <div class="modal-body">
+              <table class="table">
+                <thead>
+                  <tr>
+                    <th style="width: 20%;">Keys</th>
+                    <th>Action</th>
+                  </tr>
+                </thead>
+                <tbody>
+                  <tr>
+                    <td class="help shortcut"><kbd>?</kbd></td>
+                    <td>Open this help</td>
+                  </tr>
+                  <tr>
+                    <td class="next shortcut"><kbd>n</kbd></td>
+                    <td>Next page</td>
+                  </tr>
+                  <tr>
+                    <td class="prev shortcut"><kbd>p</kbd></td>
+                    <td>Previous page</td>
+                  </tr>
+                  <tr>
+                    <td class="search shortcut"><kbd>s</kbd></td>
+                    <td>Search</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+            <div class="modal-footer">
+            </div>
+        </div>
+    </div>
+</div>
+
+    </body>
+</html>
diff --git a/index.html b/index.html
index a861404..4db5a2b 100644
--- a/index.html
+++ b/index.html
@@ -49,6 +49,48 @@
                             <li class="active">
                                 <a href=".">Home</a>
                             </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Git <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="git/git-basics/">Git Basics</a>
+</li>
+                                    
+<li >
+    <a href="git/branches/">Working With Branches</a>
+</li>
+                                    
+<li >
+    <a href="git/github-hooks/">Github and Hooks</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Python and Web <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="python_web/intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="python_web/python-concepts/">Some Python Concepts</a>
+</li>
+                                    
+<li >
+    <a href="python_web/python-web-flask/">Python, Web and Flask</a>
+</li>
+                                    
+<li >
+    <a href="python_web/url-shorten-app/">The URL Shortening App</a>
+</li>
+                                    
+<li >
+    <a href="python_web/sre-conclusion/">SRE Aspects of The App and Conclusion</a>
+</li>
+                                </ul>
+                            </li>
                             <li class="dropdown">
                                 <a href="#" class="dropdown-toggle" data-toggle="dropdown">Systems Design <b class="caret"></b></a>
                                 <ul class="dropdown-menu">
@@ -88,7 +130,7 @@
                                 </a>
                             </li>
                             <li >
-                                <a rel="prev" href="systems_design/intro/">
+                                <a rel="prev" href="git/git-basics/">
                                     Next <i class="fa fa-arrow-right"></i>
                                 </a>
                             </li>
@@ -187,5 +229,5 @@
 
 <!--
 MkDocs version : 1.0.4
-Build Date UTC : 2020-11-02 10:02:28
+Build Date UTC : 2020-11-03 13:37:18
 -->
diff --git a/python_web/intro/index.html b/python_web/intro/index.html
new file mode 100644
index 0000000..40027f9
--- /dev/null
+++ b/python_web/intro/index.html
@@ -0,0 +1,332 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="utf-8">
+        <meta http-equiv="X-UA-Compatible" content="IE=edge">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        
+        
+        
+        <link rel="shortcut icon" href="../../img/favicon.ico">
+        <title>Intro - school_of_sre</title>
+        <link href="../../css/bootstrap-custom.min.css" rel="stylesheet">
+        <link href="../../css/font-awesome.min.css" rel="stylesheet">
+        <link href="../../css/base.css" rel="stylesheet">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css">
+        <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
+        <!--[if lt IE 9]>
+            <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+            <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
+        <![endif]-->
+
+        <script src="../../js/jquery-1.10.2.min.js" defer></script>
+        <script src="../../js/bootstrap-3.0.3.min.js" defer></script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
+        <script>hljs.initHighlightingOnLoad();</script> 
+    </head>
+
+    <body>
+
+        <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+            <div class="container">
+
+                <!-- Collapsed navigation -->
+                <div class="navbar-header">
+                    <!-- Expander button -->
+                    <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+                        <span class="sr-only">Toggle navigation</span>
+                        <span class="icon-bar"></span>
+                        <span class="icon-bar"></span>
+                        <span class="icon-bar"></span>
+                    </button>
+                    <a class="navbar-brand" href="../..">school_of_sre</a>
+                </div>
+
+                <!-- Expanded navigation -->
+                <div class="navbar-collapse collapse">
+                        <!-- Main navigation -->
+                        <ul class="nav navbar-nav">
+                            <li >
+                                <a href="../..">Home</a>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Git <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../git/git-basics/">Git Basics</a>
+</li>
+                                    
+<li >
+    <a href="../../git/branches/">Working With Branches</a>
+</li>
+                                    
+<li >
+    <a href="../../git/github-hooks/">Github and Hooks</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown active">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Python and Web <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li class="active">
+    <a href="./">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../python-concepts/">Some Python Concepts</a>
+</li>
+                                    
+<li >
+    <a href="../python-web-flask/">Python, Web and Flask</a>
+</li>
+                                    
+<li >
+    <a href="../url-shorten-app/">The URL Shortening App</a>
+</li>
+                                    
+<li >
+    <a href="../sre-conclusion/">SRE Aspects of The App and Conclusion</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Systems Design <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../systems_design/intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/scalability/">Scalability</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/availability/">Availability</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/fault-tolerance/">Fault Tolerance</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/conclusion/">Conclusion</a>
+</li>
+                                </ul>
+                            </li>
+                        </ul>
+
+                    <ul class="nav navbar-nav navbar-right">
+                        <li>
+                            <a href="#" data-toggle="modal" data-target="#mkdocs_search_modal">
+                                <i class="fa fa-search"></i> Search
+                            </a>
+                        </li>
+                            <li >
+                                <a rel="next" href="../../git/github-hooks/">
+                                    <i class="fa fa-arrow-left"></i> Previous
+                                </a>
+                            </li>
+                            <li >
+                                <a rel="prev" href="../python-concepts/">
+                                    Next <i class="fa fa-arrow-right"></i>
+                                </a>
+                            </li>
+                    </ul>
+                </div>
+            </div>
+        </div>
+
+        <div class="container">
+                <div class="col-md-3"><div class="bs-sidebar hidden-print affix well" role="complementary">
+    <ul class="nav bs-sidenav">
+        <li class="main active"><a href="#school-of-sre-python-and-the-web">School of SRE: Python and The Web</a></li>
+            <li><a href="#pre-reads">Pre - Reads</a></li>
+            <li><a href="#what-to-expect-from-this-training">What to expect from this training</a></li>
+            <li><a href="#what-is-not-covered-under-this-training">What is not covered under this training</a></li>
+            <li><a href="#training-content">Training Content</a></li>
+            <li><a href="#the-python-language">The Python Language</a></li>
+    </ul>
+</div></div>
+                <div class="col-md-9" role="main">
+
+<h1 id="school-of-sre-python-and-the-web">School of SRE: Python and The Web</h1>
+<h2 id="pre-reads">Pre - Reads</h2>
+<ul>
+<li>Basic understanding of python language.</li>
+<li>Basic familiarity with flask framework.</li>
+</ul>
+<h2 id="what-to-expect-from-this-training">What to expect from this training</h2>
+<p>This course is divided into two high level parts. In the first part, assuming familiarity with python language’s basic operations and syntax usage, we will dive a little deeper into understanding python as a language. We will compare python with other programming languages that you might already know like Java and C. We will also explore concepts of Python objects and with help of that, explore python features like decorators.</p>
+<p>In the second part which will revolve around the web, and also assume familiarity with the Flask framework, we will start from the socket module and work with HTTP requests. This will demystify how frameworks like flask work internally.</p>
+<p>And to introduce SRE flavour to the course, we will design, develop and deploy (in theory) a URL shortening application. We will emphasize parts of the whole process that are more important as an SRE of the said app/service.</p>
+<h2 id="what-is-not-covered-under-this-training">What is not covered under this training</h2>
+<p>Extensive knowledge of python internals and advanced python.</p>
+<h2 id="training-content">Training Content</h2>
+<h3 id="lab-environment-setup">Lab Environment Setup</h3>
+<p>Have latest version of python installed</p>
+<h3 id="toc">TOC</h3>
+<ol>
+<li>The Python Language<ol>
+<li>Some Python Concepts</li>
+<li>Python Gotchas</li>
+</ol>
+</li>
+<li>Python and Web<ol>
+<li>Sockets</li>
+<li>Flask</li>
+</ol>
+</li>
+<li>The URL Shortening App<ol>
+<li>Design</li>
+<li>Scaling The App</li>
+<li>Monitoring The App</li>
+</ol>
+</li>
+</ol>
+<h2 id="the-python-language">The Python Language</h2>
+<p>Assuming you know a little bit of C/C++ and Java, let's try to discuss the following questions in context of those two languages and python. You might have heard that C/C++ is a compiled language while python is an interpreted language. Generally, with compiled language we first compile the program and then run the executable while in case of python we run the source code directly like <code>python hello_world.py</code>. While Java, being an interpreted language, still has a separate compilation step and then its run. So what's really the difference?</p>
+<h3 id="compiled-vs-interpreted">Compiled vs. Interpreted</h3>
+<p>This might sound a little weird to you: python, in a way is a compiled language! Python has a compiler built-in! It is obvious in the case of java since we compile it using a separate command ie: <code>javac helloWorld.java</code> and it will produce a <code>.class</code> file which we know as a <em>bytecode</em>. Well, python is very similar to that. One difference here is that there is no separate compile command/binary needed to run a python program.</p>
+<p><strong>What is the difference then, between java and python?</strong>
+Well, Java's compiler is more strict and sophisticated. As you might know Java is a statically typed language. So the compiler is written in a way that it can verify types related errors during compile time. While python being a <em>dynamic</em> language, types are not known until a program is run. So in a way, python compiler is dumb (or, less strict). But there indeed is a compile step involved when a python program is run. You might have seen python bytecode files with <code>.pyc</code> extension. Here is how you can see bytecode for a given python program.</p>
+<pre><code class="bash"># Create a Hello World
+spatel1-mn1:tmp spatel1$ echo &quot;print('hello world')&quot; &gt; hello_world.py
+
+# Making sure it runs
+spatel1-mn1:tmp spatel1$ python3 hello_world.py
+hello world
+
+# The bytecode of the given program
+spatel1-mn1:tmp spatel1$ python -m dis hello_world.py
+ 1           0 LOAD_NAME                0 (print)
+             2 LOAD_CONST               0 ('hello world')
+             4 CALL_FUNCTION            1
+             6 POP_TOP
+             8 LOAD_CONST               1 (None)
+            10 RETURN_VALUE
+</code></pre>
+
+<p>Read more about dis module <a href="https://docs.python.org/3/library/dis.html">here</a></p>
+<p>Now coming to C/C++, there of course is a compiler. But the output is different than what java/python compiler would produce. Compiling a C program would produce what we also know as <em>machine code</em>. As opposed to bytecode.</p>
+<h3 id="running-the-programs">Running The Programs</h3>
+<p>We know compilation is involved in all 3 languages we are discussing. Just that the compilers are different in nature and they output different types of content. In case of C/C++, the output is machine code which can be directly read by your operating system. When you execute that program, your OS will know how exactly to run it. <strong>But this is not the case with bytecode.</strong></p>
+<p>Those bytecodes are language specific. Python has its own set of bytecode defined (more in <code>dis</code> module) and so does java. So naturally, your operating system will not know how to run it. To run this bytecode, we have something called Virtual Machines. Ie: The JVM or the Python VM (CPython, Jython). These so called Virtual Machines are the programs which can read the bytecode and run it on a given operating system. Python has multiple VMs available. Cpython is a python VM implemented in C language, similarly Jython is a Java implementation of python VM. <strong>At the end of the day, what they should be capable of is to understand python language syntax, be able to compile it to bytecode and be able to run that bytecode.</strong> You can implement a python VM in any language! (And people do so, just because it can be done)</p>
+<pre><code>                                                              The Operating System
+
+                                                              +------------------------------------+
+                                                              |                                    |
+                                                              |                                    |
+                                                              |                                    |
+hello_world.py                    Python bytecode             |         Python VM Process          |
+                                                              |                                    |
++----------------+                +----------------+          |         +----------------+         |
+|print(...       |   COMPILE      |LOAD_CONST...   |          |         |Reads bytecode  |         |
+|                +---------------&gt;+                +-------------------&gt;+line by line    |         |
+|                |                |                |          |         |and executes.   |         |
+|                |                |                |          |         |                |         |
++----------------+                +----------------+          |         +----------------+         |
+                                                              |                                    |
+                                                              |                                    |
+                                                              |                                    |
+hello_world.c                     OS Specific machinecode     |         A New Process              |
+                                                              |                                    |
++----------------+                +----------------+          |         +----------------+         |
+|void main() {   |   COMPILE      | binary contents|          |         | binary contents|         |
+|                +---------------&gt;+                +-------------------&gt;+                |         |
+|                |                |                |          |         |                |         |
+|                |                |                |          |         |                |         |
++----------------+                +----------------+          |         +----------------+         |
+                                                              |         (binary contents           |
+                                                              |         runs as is)                |
+                                                              |                                    |
+                                                              |                                    |
+                                                              +------------------------------------+
+</code></pre>
+
+<p>Two things to note for above diagram:</p>
+<ol>
+<li>Generally, when we run a python program, a python VM process is started which reads the python source code, compiles it to byte code and run it in a single step. Compiling is not a separate step. Shown only for illustration purpose.</li>
+<li>Binaries generated for C like languages are not <em>exactly</em> run as is. Since there are multiple types of binaries (eg: ELF), there are more complicated steps involved in order to run a binary but we will not go into that since all that is done at OS level.</li>
+</ol></div>
+        </div>
+
+        <footer class="col-md-12">
+            <hr>
+            <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
+        </footer>
+        <script>
+            var base_url = "../..",
+                shortcuts = {"search": 83, "next": 78, "help": 191, "previous": 80};
+        </script>
+        <script src="../../js/base.js" defer></script>
+        <script src="../../search/main.js" defer></script>
+
+        <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="Search Modal" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+                <h4 class="modal-title" id="exampleModalLabel">Search</h4>
+            </div>
+            <div class="modal-body">
+                <p>
+                    From here you can search these documents. Enter
+                    your search terms below.
+                </p>
+                <form role="form">
+                    <div class="form-group">
+                        <input type="text" class="form-control" placeholder="Search..." id="mkdocs-search-query" title="Type search term here">
+                    </div>
+                </form>
+                <div id="mkdocs-search-results"></div>
+            </div>
+            <div class="modal-footer">
+            </div>
+        </div>
+    </div>
+</div><div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="Keyboard Shortcuts Modal" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+                <h4 class="modal-title" id="exampleModalLabel">Keyboard Shortcuts</h4>
+            </div>
+            <div class="modal-body">
+              <table class="table">
+                <thead>
+                  <tr>
+                    <th style="width: 20%;">Keys</th>
+                    <th>Action</th>
+                  </tr>
+                </thead>
+                <tbody>
+                  <tr>
+                    <td class="help shortcut"><kbd>?</kbd></td>
+                    <td>Open this help</td>
+                  </tr>
+                  <tr>
+                    <td class="next shortcut"><kbd>n</kbd></td>
+                    <td>Next page</td>
+                  </tr>
+                  <tr>
+                    <td class="prev shortcut"><kbd>p</kbd></td>
+                    <td>Previous page</td>
+                  </tr>
+                  <tr>
+                    <td class="search shortcut"><kbd>s</kbd></td>
+                    <td>Search</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+            <div class="modal-footer">
+            </div>
+        </div>
+    </div>
+</div>
+
+    </body>
+</html>
diff --git a/python_web/python-concepts/index.html b/python_web/python-concepts/index.html
new file mode 100644
index 0000000..844e566
--- /dev/null
+++ b/python_web/python-concepts/index.html
@@ -0,0 +1,370 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="utf-8">
+        <meta http-equiv="X-UA-Compatible" content="IE=edge">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        
+        
+        
+        <link rel="shortcut icon" href="../../img/favicon.ico">
+        <title>Some Python Concepts - school_of_sre</title>
+        <link href="../../css/bootstrap-custom.min.css" rel="stylesheet">
+        <link href="../../css/font-awesome.min.css" rel="stylesheet">
+        <link href="../../css/base.css" rel="stylesheet">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css">
+        <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
+        <!--[if lt IE 9]>
+            <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+            <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
+        <![endif]-->
+
+        <script src="../../js/jquery-1.10.2.min.js" defer></script>
+        <script src="../../js/bootstrap-3.0.3.min.js" defer></script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
+        <script>hljs.initHighlightingOnLoad();</script> 
+    </head>
+
+    <body>
+
+        <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+            <div class="container">
+
+                <!-- Collapsed navigation -->
+                <div class="navbar-header">
+                    <!-- Expander button -->
+                    <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+                        <span class="sr-only">Toggle navigation</span>
+                        <span class="icon-bar"></span>
+                        <span class="icon-bar"></span>
+                        <span class="icon-bar"></span>
+                    </button>
+                    <a class="navbar-brand" href="../..">school_of_sre</a>
+                </div>
+
+                <!-- Expanded navigation -->
+                <div class="navbar-collapse collapse">
+                        <!-- Main navigation -->
+                        <ul class="nav navbar-nav">
+                            <li >
+                                <a href="../..">Home</a>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Git <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../git/git-basics/">Git Basics</a>
+</li>
+                                    
+<li >
+    <a href="../../git/branches/">Working With Branches</a>
+</li>
+                                    
+<li >
+    <a href="../../git/github-hooks/">Github and Hooks</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown active">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Python and Web <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../intro/">Intro</a>
+</li>
+                                    
+<li class="active">
+    <a href="./">Some Python Concepts</a>
+</li>
+                                    
+<li >
+    <a href="../python-web-flask/">Python, Web and Flask</a>
+</li>
+                                    
+<li >
+    <a href="../url-shorten-app/">The URL Shortening App</a>
+</li>
+                                    
+<li >
+    <a href="../sre-conclusion/">SRE Aspects of The App and Conclusion</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Systems Design <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../systems_design/intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/scalability/">Scalability</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/availability/">Availability</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/fault-tolerance/">Fault Tolerance</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/conclusion/">Conclusion</a>
+</li>
+                                </ul>
+                            </li>
+                        </ul>
+
+                    <ul class="nav navbar-nav navbar-right">
+                        <li>
+                            <a href="#" data-toggle="modal" data-target="#mkdocs_search_modal">
+                                <i class="fa fa-search"></i> Search
+                            </a>
+                        </li>
+                            <li >
+                                <a rel="next" href="../intro/">
+                                    <i class="fa fa-arrow-left"></i> Previous
+                                </a>
+                            </li>
+                            <li >
+                                <a rel="prev" href="../python-web-flask/">
+                                    Next <i class="fa fa-arrow-right"></i>
+                                </a>
+                            </li>
+                    </ul>
+                </div>
+            </div>
+        </div>
+
+        <div class="container">
+                <div class="col-md-3"><div class="bs-sidebar hidden-print affix well" role="complementary">
+    <ul class="nav bs-sidenav">
+        <li class="main active"><a href="#some-python-concepts">Some Python Concepts</a></li>
+            <li><a href="#python-functions">Python Functions</a></li>
+            <li><a href="#decorators">Decorators</a></li>
+            <li><a href="#some-gotchas">Some Gotchas</a></li>
+    </ul>
+</div></div>
+                <div class="col-md-9" role="main">
+
+<h1 id="some-python-concepts">Some Python Concepts</h1>
+<p>Though you are expected to know python and its syntax at basic level, let us discuss some fundamental concepts that will help you understand the python language better.</p>
+<p><strong>Everything in Python is an object.</strong></p>
+<p>That includes the functions, lists, dicts, classes, modules, a running function (instance of function definition), everything. In the CPython, it would mean there is an underlying struct variable for each object.</p>
+<p>In python's current execution context, all the variables are stored in a dict. It'd be a string to object mapping. If you have a function and a float variable defined in the current context, here is how it is handled internally.</p>
+<pre><code class="python">&gt;&gt;&gt; float_number=42.0
+&gt;&gt;&gt; def foo_func():
+...     pass
+...
+
+# NOTICE HOW VARIABLE NAMES ARE STRINGS, stored in a dict
+&gt;&gt;&gt; locals()
+{'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': &lt;class '_frozen_importlib.BuiltinImporter'&gt;, '__spec__': None, '__annotations__': {}, '__builtins__': &lt;module 'builtins' (built-in)&gt;, 'float_number': 42.0, 'foo_func': &lt;function foo_func at 0x1055847a0&gt;}
+</code></pre>
+
+<h2 id="python-functions">Python Functions</h2>
+<p>Since functions too are objects, we can see what all attributes a function contains as following</p>
+<pre><code class="python">&gt;&gt;&gt; def hello(name):
+...     print(f&quot;Hello, {name}!&quot;)
+...
+&gt;&gt;&gt; dir(hello)
+['__annotations__', '__call__', '__class__', '__closure__', '__code__', '__defaults__', '__delattr__', '__dict__',
+'__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__get__', '__getattribute__', '__globals__', '__gt__',
+'__hash__', '__init__', '__init_subclass__', '__kwdefaults__', '__le__', '__lt__', '__module__', '__name__',
+'__ne__', '__new__', '__qualname__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__',
+'__subclasshook__']
+</code></pre>
+
+<p>While there are a lot of them, let's look at some interesting ones</p>
+<h4 id="globals"><strong>globals</strong></h4>
+<p>This attribute, as the name suggests, has references of global variables. If you ever need to know what all global variables are in the scope of this function, this will tell you. See how the function start seeing the new variable in globals</p>
+<pre><code class="python">&gt;&gt;&gt; hello.__globals__
+{'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': &lt;class '_frozen_importlib.BuiltinImporter'&gt;, '__spec__': None, '__annotations__': {}, '__builtins__': &lt;module 'builtins' (built-in)&gt;, 'hello': &lt;function hello at 0x7fe4e82554c0&gt;}
+
+# adding new global variable
+&gt;&gt;&gt; GLOBAL=&quot;g_val&quot;
+&gt;&gt;&gt; hello.__globals__
+{'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': &lt;class '_frozen_importlib.BuiltinImporter'&gt;, '__spec__': None, '__annotations__': {}, '__builtins__': &lt;module 'builtins' (built-in)&gt;, 'hello': &lt;function hello at 0x7fe4e82554c0&gt;, 'GLOBAL': 'g_val'}
+</code></pre>
+
+<h3 id="code"><strong>code</strong></h3>
+<p>This is an interesting one! As everything in python is an object, this includes the bytecode too. The compiled python bytecode is a python code object. Which is accessible via <code>__code__</code> attribute here. A function has an associated code object which carries some interesting information.</p>
+<pre><code class="python"># the file in which function is defined
+# stdin here since this is run in an interpreter
+&gt;&gt;&gt; hello.__code__.co_filename
+'&lt;stdin&gt;'
+
+# number of arguments the function takes
+&gt;&gt;&gt; hello.__code__.co_argcount
+1
+
+# local variable names
+&gt;&gt;&gt; hello.__code__.co_varnames
+('name',)
+
+# the function code's compiled bytecode
+&gt;&gt;&gt; hello.__code__.co_code
+b't\x00d\x01|\x00\x9b\x00d\x02\x9d\x03\x83\x01\x01\x00d\x00S\x00'
+</code></pre>
+
+<p>There are more code attributes which you can enlist by <code>&gt;&gt;&gt; dir(hello.__code__)</code></p>
+<h2 id="decorators">Decorators</h2>
+<p>Related to functions, python has another feature called decorators. Let's see how that works, keeping  <code>everything is an object</code> in mind.</p>
+<p>Here is a sample decorator:</p>
+<pre><code class="python">&gt;&gt;&gt; def deco(func):
+...     def inner():
+...             print(&quot;before&quot;)
+...             func()
+...             print(&quot;after&quot;)
+...     return inner
+...
+&gt;&gt;&gt; @deco
+... def hello_world():
+...     print(&quot;hello world&quot;)
+...
+&gt;&gt;&gt;
+&gt;&gt;&gt; hello_world()
+before
+hello world
+after
+</code></pre>
+
+<p>Here <code>@deco</code> syntax is used to decorate the <code>hello_world</code> function. It is essentially same as doing</p>
+<pre><code class="python">&gt;&gt;&gt; def hello_world():
+...     print(&quot;hello world&quot;)
+...
+&gt;&gt;&gt; hello_world = deco(hello_world)
+</code></pre>
+
+<p>What goes inside the <code>deco</code> function might seem complex. Let's try to uncover it.</p>
+<ol>
+<li>Function <code>hello_world</code> is created</li>
+<li>It is passed to <code>deco</code> function</li>
+<li><code>deco</code> create a new function<ol>
+<li>This new function is calls <code>hello_world</code> function</li>
+<li>And does a couple other things</li>
+</ol>
+</li>
+<li><code>deco</code> returns the newly created function</li>
+<li><code>hello_world</code> is replaced with above function</li>
+</ol>
+<p>Let's visualize it for better understanding</p>
+<pre><code>       BEFORE                   function_object (ID: 100)
+
+       &quot;hello_world&quot;            +--------------------+
+               +                |print(&quot;hello_world&quot;)|
+               |                |                    |
+               +--------------&gt; |                    |
+                                |                    |
+                                +--------------------+
+
+
+       WHAT DECORATOR DOES
+
+       creates a new function (ID: 101)
+       +---------------------------------+
+       |input arg: function with id: 100 |
+       |                                 |
+       |print(&quot;before&quot;)                  |
+       |call function object with id 100 |
+       |print(&quot;after&quot;)                   |
+       |                                 |
+       +---------------------------^-----+
+                                   |
+                                   |
+       AFTER                       |
+                                   |
+                                   |
+       &quot;hello_world&quot; +-------------+
+</code></pre>
+
+<p>Note how the <code>hello_world</code> name points to a new function object but that new function object knows the reference (ID) of the original function.</p>
+<h2 id="some-gotchas">Some Gotchas</h2>
+<ul>
+<li>While it is very quick to build prototypes in python and there are tons of libraries available, as the codebase complexity increases, type errors become more common and will get hard to deal with. (There are solutions to that problem like type annotations in python. Checkout <a href="http://mypy-lang.org/">mypy</a>.)</li>
+<li>Because python is dynamically typed language, that means all types are determined at runtime. And that makes python run very slow compared to other statically typed languages.</li>
+<li>Python has something called <a href="https://www.dabeaz.com/python/UnderstandingGIL.pdf">GIL</a> (global interpreter lock) which is a limiting factor for utilizing multiple CPI cores for parallel computation.</li>
+<li>Some weird things that python does: https://github.com/satwikkansal/wtfpython</li>
+</ul></div>
+        </div>
+
+        <footer class="col-md-12">
+            <hr>
+            <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
+        </footer>
+        <script>
+            var base_url = "../..",
+                shortcuts = {"search": 83, "next": 78, "help": 191, "previous": 80};
+        </script>
+        <script src="../../js/base.js" defer></script>
+        <script src="../../search/main.js" defer></script>
+
+        <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="Search Modal" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+                <h4 class="modal-title" id="exampleModalLabel">Search</h4>
+            </div>
+            <div class="modal-body">
+                <p>
+                    From here you can search these documents. Enter
+                    your search terms below.
+                </p>
+                <form role="form">
+                    <div class="form-group">
+                        <input type="text" class="form-control" placeholder="Search..." id="mkdocs-search-query" title="Type search term here">
+                    </div>
+                </form>
+                <div id="mkdocs-search-results"></div>
+            </div>
+            <div class="modal-footer">
+            </div>
+        </div>
+    </div>
+</div><div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="Keyboard Shortcuts Modal" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+                <h4 class="modal-title" id="exampleModalLabel">Keyboard Shortcuts</h4>
+            </div>
+            <div class="modal-body">
+              <table class="table">
+                <thead>
+                  <tr>
+                    <th style="width: 20%;">Keys</th>
+                    <th>Action</th>
+                  </tr>
+                </thead>
+                <tbody>
+                  <tr>
+                    <td class="help shortcut"><kbd>?</kbd></td>
+                    <td>Open this help</td>
+                  </tr>
+                  <tr>
+                    <td class="next shortcut"><kbd>n</kbd></td>
+                    <td>Next page</td>
+                  </tr>
+                  <tr>
+                    <td class="prev shortcut"><kbd>p</kbd></td>
+                    <td>Previous page</td>
+                  </tr>
+                  <tr>
+                    <td class="search shortcut"><kbd>s</kbd></td>
+                    <td>Search</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+            <div class="modal-footer">
+            </div>
+        </div>
+    </div>
+</div>
+
+    </body>
+</html>
diff --git a/python_web/python-web-flask/index.html b/python_web/python-web-flask/index.html
new file mode 100644
index 0000000..4c1278a
--- /dev/null
+++ b/python_web/python-web-flask/index.html
@@ -0,0 +1,270 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="utf-8">
+        <meta http-equiv="X-UA-Compatible" content="IE=edge">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        
+        
+        
+        <link rel="shortcut icon" href="../../img/favicon.ico">
+        <title>Python, Web and Flask - school_of_sre</title>
+        <link href="../../css/bootstrap-custom.min.css" rel="stylesheet">
+        <link href="../../css/font-awesome.min.css" rel="stylesheet">
+        <link href="../../css/base.css" rel="stylesheet">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css">
+        <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
+        <!--[if lt IE 9]>
+            <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+            <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
+        <![endif]-->
+
+        <script src="../../js/jquery-1.10.2.min.js" defer></script>
+        <script src="../../js/bootstrap-3.0.3.min.js" defer></script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
+        <script>hljs.initHighlightingOnLoad();</script> 
+    </head>
+
+    <body>
+
+        <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+            <div class="container">
+
+                <!-- Collapsed navigation -->
+                <div class="navbar-header">
+                    <!-- Expander button -->
+                    <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+                        <span class="sr-only">Toggle navigation</span>
+                        <span class="icon-bar"></span>
+                        <span class="icon-bar"></span>
+                        <span class="icon-bar"></span>
+                    </button>
+                    <a class="navbar-brand" href="../..">school_of_sre</a>
+                </div>
+
+                <!-- Expanded navigation -->
+                <div class="navbar-collapse collapse">
+                        <!-- Main navigation -->
+                        <ul class="nav navbar-nav">
+                            <li >
+                                <a href="../..">Home</a>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Git <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../git/git-basics/">Git Basics</a>
+</li>
+                                    
+<li >
+    <a href="../../git/branches/">Working With Branches</a>
+</li>
+                                    
+<li >
+    <a href="../../git/github-hooks/">Github and Hooks</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown active">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Python and Web <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../python-concepts/">Some Python Concepts</a>
+</li>
+                                    
+<li class="active">
+    <a href="./">Python, Web and Flask</a>
+</li>
+                                    
+<li >
+    <a href="../url-shorten-app/">The URL Shortening App</a>
+</li>
+                                    
+<li >
+    <a href="../sre-conclusion/">SRE Aspects of The App and Conclusion</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Systems Design <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../systems_design/intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/scalability/">Scalability</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/availability/">Availability</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/fault-tolerance/">Fault Tolerance</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/conclusion/">Conclusion</a>
+</li>
+                                </ul>
+                            </li>
+                        </ul>
+
+                    <ul class="nav navbar-nav navbar-right">
+                        <li>
+                            <a href="#" data-toggle="modal" data-target="#mkdocs_search_modal">
+                                <i class="fa fa-search"></i> Search
+                            </a>
+                        </li>
+                            <li >
+                                <a rel="next" href="../python-concepts/">
+                                    <i class="fa fa-arrow-left"></i> Previous
+                                </a>
+                            </li>
+                            <li >
+                                <a rel="prev" href="../url-shorten-app/">
+                                    Next <i class="fa fa-arrow-right"></i>
+                                </a>
+                            </li>
+                    </ul>
+                </div>
+            </div>
+        </div>
+
+        <div class="container">
+                <div class="col-md-3"><div class="bs-sidebar hidden-print affix well" role="complementary">
+    <ul class="nav bs-sidenav">
+        <li class="main active"><a href="#python-web-amd-flask">Python, Web amd Flask</a></li>
+            <li><a href="#sockets">Sockets</a></li>
+            <li><a href="#flask">Flask</a></li>
+    </ul>
+</div></div>
+                <div class="col-md-9" role="main">
+
+<h1 id="python-web-amd-flask">Python, Web amd Flask</h1>
+<p>Back in the old days, websites were simple. They were simple static html contents. A webserver would be listening on a defined port and according to the HTTP request received, it would read files from disk and return them in response. But since then, complexity has evolved and websites are now dynamic. Depending on the request, multiple operations need to be performed like reading from database or calling other API and finally returning some response (HTML data, JSON content etc.)</p>
+<p>Since serving web requests is no longer a simple task like reading files from disk and return contents, we need to process each http request, perform some operations programmatically and construct a response.</p>
+<h2 id="sockets">Sockets</h2>
+<p>Though we have frameworks like flask, HTTP is still a protocol that works over TCP protocol. So let us setup a TCP server and send an HTTP request and inspect the request's payload. Note that this is not a tutorial on socket programming but what we are doing here is inspecting HTTP protocol at its ground level and look at what its contents look like. (Ref: <a href="https://realpython.com/python-sockets/">Socket Programming in Python (Guide) on RealPython</a>)</p>
+<pre><code class="python">import socket
+
+HOST = '127.0.0.1'  # Standard loopback interface address (localhost)
+PORT = 65432        # Port to listen on (non-privileged ports are &gt; 1023)
+
+with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+   s.bind((HOST, PORT))
+   s.listen()
+   conn, addr = s.accept()
+   with conn:
+       print('Connected by', addr)
+       while True:
+           data = conn.recv(1024)
+           if not data:
+               break
+           print(data)
+</code></pre>
+
+<p>Then we open <code>localhost:65432</code> in our web browser and following would be the output:</p>
+<pre><code class="bash">Connected by ('127.0.0.1', 54719)
+b'GET / HTTP/1.1\r\nHost: localhost:65432\r\nConnection: keep-alive\r\nDNT: 1\r\nUpgrade-Insecure-Requests: 1\r\nUser-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36 Edg/85.0.564.44\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9\r\nSec-Fetch-Site: none\r\nSec-Fetch-Mode: navigate\r\nSec-Fetch-User: ?1\r\nSec-Fetch-Dest: document\r\nAccept-Encoding: gzip, deflate, br\r\nAccept-Language: en-US,en;q=0.9\r\n\r\n'
+</code></pre>
+
+<p>Examine closely and the content will look like the HTTP protocol's format. ie:</p>
+<pre><code>HTTP_METHOD URI_PATH HTTP_VERSION
+HEADERS_SEPARATED_BY_SEPARATOR
+</code></pre>
+
+<p>So though it's a blob of bytes, knowing <a href="https://tools.ietf.org/html/rfc2616">http protocol specification</a>, you can parse that string (ie: split by <code>\r\n</code>) and get meaningful information out of it.</p>
+<h2 id="flask">Flask</h2>
+<p>Flask, and other such frameworks does pretty much what we just discussed in the last section (with added more sophistication). They listen on a port on a TCP socket, receive an HTTP request, parse the data according to protocol format and make it available to you in a convenient manner.</p>
+<p>ie: you can access headers in flask by <code>request.headers</code> which is made available to you by splitting above payload by <code>/r/n</code>, as defined in http protocol.</p>
+<p>Another example: we register routes in flask by <code>@app.route("/hello")</code>. What flask will do is maintain a registry internally which will map <code>/hello</code> with the function you decorated with. Now whenever a request comes with the <code>/hello</code> route (second component in the first line, split by space), flask calls the registered function and returns whatever the function returned.</p>
+<p>Same with all other web frameworks in other languages too. They all work on similar principles. What they basically do is understand the HTTP protocol, parses the HTTP request data and gives us programmers a nice interface to work with HTTP requests.</p>
+<p>Not so much of magic, innit?</p></div>
+        </div>
+
+        <footer class="col-md-12">
+            <hr>
+            <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
+        </footer>
+        <script>
+            var base_url = "../..",
+                shortcuts = {"search": 83, "next": 78, "help": 191, "previous": 80};
+        </script>
+        <script src="../../js/base.js" defer></script>
+        <script src="../../search/main.js" defer></script>
+
+        <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="Search Modal" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+                <h4 class="modal-title" id="exampleModalLabel">Search</h4>
+            </div>
+            <div class="modal-body">
+                <p>
+                    From here you can search these documents. Enter
+                    your search terms below.
+                </p>
+                <form role="form">
+                    <div class="form-group">
+                        <input type="text" class="form-control" placeholder="Search..." id="mkdocs-search-query" title="Type search term here">
+                    </div>
+                </form>
+                <div id="mkdocs-search-results"></div>
+            </div>
+            <div class="modal-footer">
+            </div>
+        </div>
+    </div>
+</div><div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="Keyboard Shortcuts Modal" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+                <h4 class="modal-title" id="exampleModalLabel">Keyboard Shortcuts</h4>
+            </div>
+            <div class="modal-body">
+              <table class="table">
+                <thead>
+                  <tr>
+                    <th style="width: 20%;">Keys</th>
+                    <th>Action</th>
+                  </tr>
+                </thead>
+                <tbody>
+                  <tr>
+                    <td class="help shortcut"><kbd>?</kbd></td>
+                    <td>Open this help</td>
+                  </tr>
+                  <tr>
+                    <td class="next shortcut"><kbd>n</kbd></td>
+                    <td>Next page</td>
+                  </tr>
+                  <tr>
+                    <td class="prev shortcut"><kbd>p</kbd></td>
+                    <td>Previous page</td>
+                  </tr>
+                  <tr>
+                    <td class="search shortcut"><kbd>s</kbd></td>
+                    <td>Search</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+            <div class="modal-footer">
+            </div>
+        </div>
+    </div>
+</div>
+
+    </body>
+</html>
diff --git a/python_web/sre-conclusion/index.html b/python_web/sre-conclusion/index.html
new file mode 100644
index 0000000..3260933
--- /dev/null
+++ b/python_web/sre-conclusion/index.html
@@ -0,0 +1,263 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="utf-8">
+        <meta http-equiv="X-UA-Compatible" content="IE=edge">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        
+        
+        
+        <link rel="shortcut icon" href="../../img/favicon.ico">
+        <title>SRE Aspects of The App and Conclusion - school_of_sre</title>
+        <link href="../../css/bootstrap-custom.min.css" rel="stylesheet">
+        <link href="../../css/font-awesome.min.css" rel="stylesheet">
+        <link href="../../css/base.css" rel="stylesheet">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css">
+        <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
+        <!--[if lt IE 9]>
+            <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+            <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
+        <![endif]-->
+
+        <script src="../../js/jquery-1.10.2.min.js" defer></script>
+        <script src="../../js/bootstrap-3.0.3.min.js" defer></script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
+        <script>hljs.initHighlightingOnLoad();</script> 
+    </head>
+
+    <body>
+
+        <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+            <div class="container">
+
+                <!-- Collapsed navigation -->
+                <div class="navbar-header">
+                    <!-- Expander button -->
+                    <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+                        <span class="sr-only">Toggle navigation</span>
+                        <span class="icon-bar"></span>
+                        <span class="icon-bar"></span>
+                        <span class="icon-bar"></span>
+                    </button>
+                    <a class="navbar-brand" href="../..">school_of_sre</a>
+                </div>
+
+                <!-- Expanded navigation -->
+                <div class="navbar-collapse collapse">
+                        <!-- Main navigation -->
+                        <ul class="nav navbar-nav">
+                            <li >
+                                <a href="../..">Home</a>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Git <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../git/git-basics/">Git Basics</a>
+</li>
+                                    
+<li >
+    <a href="../../git/branches/">Working With Branches</a>
+</li>
+                                    
+<li >
+    <a href="../../git/github-hooks/">Github and Hooks</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown active">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Python and Web <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../python-concepts/">Some Python Concepts</a>
+</li>
+                                    
+<li >
+    <a href="../python-web-flask/">Python, Web and Flask</a>
+</li>
+                                    
+<li >
+    <a href="../url-shorten-app/">The URL Shortening App</a>
+</li>
+                                    
+<li class="active">
+    <a href="./">SRE Aspects of The App and Conclusion</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Systems Design <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../systems_design/intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/scalability/">Scalability</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/availability/">Availability</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/fault-tolerance/">Fault Tolerance</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/conclusion/">Conclusion</a>
+</li>
+                                </ul>
+                            </li>
+                        </ul>
+
+                    <ul class="nav navbar-nav navbar-right">
+                        <li>
+                            <a href="#" data-toggle="modal" data-target="#mkdocs_search_modal">
+                                <i class="fa fa-search"></i> Search
+                            </a>
+                        </li>
+                            <li >
+                                <a rel="next" href="../url-shorten-app/">
+                                    <i class="fa fa-arrow-left"></i> Previous
+                                </a>
+                            </li>
+                            <li >
+                                <a rel="prev" href="../../systems_design/intro/">
+                                    Next <i class="fa fa-arrow-right"></i>
+                                </a>
+                            </li>
+                    </ul>
+                </div>
+            </div>
+        </div>
+
+        <div class="container">
+                <div class="col-md-3"><div class="bs-sidebar hidden-print affix well" role="complementary">
+    <ul class="nav bs-sidenav">
+        <li class="main active"><a href="#sre-parts-of-the-app-and-conclusion">SRE Parts of The App and Conclusion</a></li>
+            <li><a href="#scaling-the-app">Scaling The App</a></li>
+            <li><a href="#monitoring-strategy">Monitoring Strategy</a></li>
+            <li><a href="#sre-use-cases">SRE Use-cases</a></li>
+            <li><a href="#optional-exercises">Optional Exercises</a></li>
+            <li><a href="#conclusion">Conclusion</a></li>
+    </ul>
+</div></div>
+                <div class="col-md-9" role="main">
+
+<h1 id="sre-parts-of-the-app-and-conclusion">SRE Parts of The App and Conclusion</h1>
+<h2 id="scaling-the-app">Scaling The App</h2>
+<p>The design and development is just a part of the journey. We will need to setup continuous integration and continuous delivery pipelines sooner or later. And we have to deploy this app somewhere.</p>
+<p>Initially we can start with deploying this app on one virtual machine on any cloud provider. But this is a <code>Single point of failure</code> which is something we never allow as an SRE (or even as an engineer). So an improvement here can be having multiple instances of applications deployed behind a load balancer. This certainly prevents problems of one machine going down.</p>
+<p>Scaling here would mean adding more instances behind the load balancer. But this is scalable upto only a certain point. After that, other bottlenecks in the system will start appearing. ie: DB will become the bottleneck, or perhaps the load balancer itself. How do you know what is the bottleneck? You need to have observability into each aspects of the application architecture.</p>
+<p>Only after you have metrics, you will be able to know what is going wrong where. <strong>What gets measured, gets fixed!</strong></p>
+<p>Get deeper insights into scaling from School Of SRE's Scalability module and post going through it, apply your learnings and takeaways to this app. Think how will we make this app geographically distributed and highly available and scalable.</p>
+<h2 id="monitoring-strategy">Monitoring Strategy</h2>
+<p>Once we have our application deployed. It will be working ok. But not forever. Reliability is in the title of our job and we make systems reliable by making the design in a certain way. But things still will go down. Machines will fail. Disks will behave weirdly. Buggy code will get pushed to production. And all these possible scenarios will make the system less reliable. So what do we do? <strong>We monitor!</strong></p>
+<p>We keep an eye on the system's health and if anything is not going as expected, we want ourselves to get alerted.</p>
+<p>Now let's think in terms of the given url shortening app. We need to monitor it. And we would want to get notified in case something goes wrong. But we first need to decide what is that <em>something</em> that we want to keep an eye on.</p>
+<ol>
+<li>Since it's a web app serving HTTP requests, we want to keep an eye on HTTP Status codes and latencies</li>
+<li>Request volume again is a good candidate, if the app is receiving an unusual amount of traffic, something might be off.</li>
+<li>We also want to keep an eye on the database so depending on the database solution chosen. Query times, volumes, disk usage etc.</li>
+<li>Finally, there also needs to be some external monitoring which runs periodic tests from devices outside of your data centers. This emulates customers and ensures that from customer point of view, the system is working as expected.</li>
+</ol>
+<h2 id="sre-use-cases">SRE Use-cases</h2>
+<p>In the world of SRE, python is a widely used language. For small scripts and tooling developed for various purposes. Since tooling developed by SRE works with critical pieces of infrastructure and has great power (to bring things down), it is important to know what you are doing while using a programming language and its features. Also it is equally important to know the language and its characteristics while debugging the issues. As an SRE having a deeper understanding of python language, it has helped me a lot to debug very sneaky bugs and be generally more aware and informed while making certain design decisions.</p>
+<p>While developing tools may or may not be part of SRE job, supporting tools or services is more likely to be a daily duty. Building an application or tool is just a small part of productionization. While there is certainly that goes in the design of the application itself to make it more robust, as an SRE you are responsible for its reliability and stability once it is deployed and running. And to ensure that, you’d need to understand the application first and then come up with a strategy to monitor it properly and be prepared for various failure scenarios.</p>
+<h2 id="optional-exercises">Optional Exercises</h2>
+<ol>
+<li>Make a decorator that will cache function return values depending on input parameters.</li>
+<li>Host the URL shortening app on any cloud provider.</li>
+<li>Setup monitoring using many of the tools available like catchpoint, datadog etc.</li>
+<li>Create a minimal flask-like framework on top of TCP sockets.</li>
+</ol>
+<h2 id="conclusion">Conclusion</h2>
+<p>This module, in the first part, aims to make you more aware of the things that will happen when you choose python as your programming language and what happens when you run a python program. With the knowledge of how python handles things internally as objects, lot of seemingly magic things in python will start to make more sense.</p>
+<p>The second part will first explain how a framework like flask works using the existing knowledge of protocols like TCP and HTTP. It then touches the whole lifecycle of an application development lifecycle including the SRE parts of it. While the design and areas in architecture considered will not be exhaustive, it will give a good overview of things that are also important being an SRE and why they are important.</p></div>
+        </div>
+
+        <footer class="col-md-12">
+            <hr>
+            <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
+        </footer>
+        <script>
+            var base_url = "../..",
+                shortcuts = {"search": 83, "next": 78, "help": 191, "previous": 80};
+        </script>
+        <script src="../../js/base.js" defer></script>
+        <script src="../../search/main.js" defer></script>
+
+        <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="Search Modal" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+                <h4 class="modal-title" id="exampleModalLabel">Search</h4>
+            </div>
+            <div class="modal-body">
+                <p>
+                    From here you can search these documents. Enter
+                    your search terms below.
+                </p>
+                <form role="form">
+                    <div class="form-group">
+                        <input type="text" class="form-control" placeholder="Search..." id="mkdocs-search-query" title="Type search term here">
+                    </div>
+                </form>
+                <div id="mkdocs-search-results"></div>
+            </div>
+            <div class="modal-footer">
+            </div>
+        </div>
+    </div>
+</div><div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="Keyboard Shortcuts Modal" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+                <h4 class="modal-title" id="exampleModalLabel">Keyboard Shortcuts</h4>
+            </div>
+            <div class="modal-body">
+              <table class="table">
+                <thead>
+                  <tr>
+                    <th style="width: 20%;">Keys</th>
+                    <th>Action</th>
+                  </tr>
+                </thead>
+                <tbody>
+                  <tr>
+                    <td class="help shortcut"><kbd>?</kbd></td>
+                    <td>Open this help</td>
+                  </tr>
+                  <tr>
+                    <td class="next shortcut"><kbd>n</kbd></td>
+                    <td>Next page</td>
+                  </tr>
+                  <tr>
+                    <td class="prev shortcut"><kbd>p</kbd></td>
+                    <td>Previous page</td>
+                  </tr>
+                  <tr>
+                    <td class="search shortcut"><kbd>s</kbd></td>
+                    <td>Search</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+            <div class="modal-footer">
+            </div>
+        </div>
+    </div>
+</div>
+
+    </body>
+</html>
diff --git a/python_web/url-shorten-app/index.html b/python_web/url-shorten-app/index.html
new file mode 100644
index 0000000..1bec57e
--- /dev/null
+++ b/python_web/url-shorten-app/index.html
@@ -0,0 +1,332 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="utf-8">
+        <meta http-equiv="X-UA-Compatible" content="IE=edge">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        
+        
+        
+        <link rel="shortcut icon" href="../../img/favicon.ico">
+        <title>The URL Shortening App - school_of_sre</title>
+        <link href="../../css/bootstrap-custom.min.css" rel="stylesheet">
+        <link href="../../css/font-awesome.min.css" rel="stylesheet">
+        <link href="../../css/base.css" rel="stylesheet">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css">
+        <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
+        <!--[if lt IE 9]>
+            <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+            <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
+        <![endif]-->
+
+        <script src="../../js/jquery-1.10.2.min.js" defer></script>
+        <script src="../../js/bootstrap-3.0.3.min.js" defer></script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
+        <script>hljs.initHighlightingOnLoad();</script> 
+    </head>
+
+    <body>
+
+        <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+            <div class="container">
+
+                <!-- Collapsed navigation -->
+                <div class="navbar-header">
+                    <!-- Expander button -->
+                    <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+                        <span class="sr-only">Toggle navigation</span>
+                        <span class="icon-bar"></span>
+                        <span class="icon-bar"></span>
+                        <span class="icon-bar"></span>
+                    </button>
+                    <a class="navbar-brand" href="../..">school_of_sre</a>
+                </div>
+
+                <!-- Expanded navigation -->
+                <div class="navbar-collapse collapse">
+                        <!-- Main navigation -->
+                        <ul class="nav navbar-nav">
+                            <li >
+                                <a href="../..">Home</a>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Git <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../git/git-basics/">Git Basics</a>
+</li>
+                                    
+<li >
+    <a href="../../git/branches/">Working With Branches</a>
+</li>
+                                    
+<li >
+    <a href="../../git/github-hooks/">Github and Hooks</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown active">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Python and Web <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../python-concepts/">Some Python Concepts</a>
+</li>
+                                    
+<li >
+    <a href="../python-web-flask/">Python, Web and Flask</a>
+</li>
+                                    
+<li class="active">
+    <a href="./">The URL Shortening App</a>
+</li>
+                                    
+<li >
+    <a href="../sre-conclusion/">SRE Aspects of The App and Conclusion</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Systems Design <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../systems_design/intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/scalability/">Scalability</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/availability/">Availability</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/fault-tolerance/">Fault Tolerance</a>
+</li>
+                                    
+<li >
+    <a href="../../systems_design/conclusion/">Conclusion</a>
+</li>
+                                </ul>
+                            </li>
+                        </ul>
+
+                    <ul class="nav navbar-nav navbar-right">
+                        <li>
+                            <a href="#" data-toggle="modal" data-target="#mkdocs_search_modal">
+                                <i class="fa fa-search"></i> Search
+                            </a>
+                        </li>
+                            <li >
+                                <a rel="next" href="../python-web-flask/">
+                                    <i class="fa fa-arrow-left"></i> Previous
+                                </a>
+                            </li>
+                            <li >
+                                <a rel="prev" href="../sre-conclusion/">
+                                    Next <i class="fa fa-arrow-right"></i>
+                                </a>
+                            </li>
+                    </ul>
+                </div>
+            </div>
+        </div>
+
+        <div class="container">
+                <div class="col-md-3"><div class="bs-sidebar hidden-print affix well" role="complementary">
+    <ul class="nav bs-sidenav">
+        <li class="main active"><a href="#the-url-shortening-app">The URL Shortening App</a></li>
+            <li><a href="#design">Design</a></li>
+    </ul>
+</div></div>
+                <div class="col-md-9" role="main">
+
+<h1 id="the-url-shortening-app">The URL Shortening App</h1>
+<p>Let's build a very simple URL shortening app using flask and try to incorporate all aspects of the development process including the reliability aspects. We will not be building the UI and we will come up with a minimal set of API that will be enough for the app to function well.</p>
+<h2 id="design">Design</h2>
+<p>We don't jump directly to coding. First thing we do is gather requirements. Come up with an approach. Have the approach/design reviewed by peers. Evolve, iterate, document the decisions and tradeoffs. And then finally implement. While we will not do the full blown design document here, we will raise certain questions here that are important to the design.</p>
+<h3 id="1-high-level-operations-and-api-endpoints">1. High Level Operations and API Endpoints</h3>
+<p>Since it's a URL shortening app, we will need an API for generating the shorten link given an original link. And an API/Endpoint which will accept the shorten link and redirect to original URL. We are not including the user aspect of the app to keep things minimal. These two API should make app functional and usable by anyone.</p>
+<h3 id="2-how-to-shorten">2. How to shorten?</h3>
+<p>Given a url, we will need to generate a shortened version of it. One approach could be using random characters for each link. Another thing that can be done is to use some sort of hashing algorithm. The benefit here is we will reuse the same hash for the same link. ie: if lot of people are shortening <code>https://www.linkedin.com</code> they all will have the same value, compared to multiple entries in DB if chosen random characters.</p>
+<p>What about hash collisions? Even in random characters approach, though there is a less probability, hash collisions can happen. And we need to be mindful of them. In that case we might want to prepend/append the string with some random value to avoid conflict.</p>
+<p>Also, choice of hash algorithm matters. We will need to analyze algorithms. Their CPU requirements and their characteristics. Choose one that suits the most.</p>
+<h3 id="3-is-url-valid">3. Is URL Valid?</h3>
+<p>Given a URL to shorten, how do we verify if the URL is valid? Do we even verify or validate? One basic check that can be done is see if the URL matches a regex of a URL. To go even further we can try opening/visiting the URL. But there are certain gotchas here.</p>
+<ol>
+<li>We need to define success criteria. ie: HTTP 200 means it is valid.</li>
+<li>What is the URL is in private network?</li>
+<li>What if URL is temporarily down?</li>
+</ol>
+<h3 id="4-storage">4. Storage</h3>
+<p>Finally, storage. Where will we store the data that we will generate over time? There are multiple database solutions available and we will need to choose the one that suits this app the most. Relational database like MySQL would be a fair choice but <strong>be sure to checkout School of SRE's database section for deeper insights into making a more informed decision.</strong></p>
+<h3 id="5-other">5. Other</h3>
+<p>We are not accounting for users into our app and other possible features like rate limiting, customized links etc but it will eventually come up with time. Depending on the requirements, they too might need to get incorporated.</p>
+<p>The minimal working code is given below for reference but I'd encourage you to come up with your own.</p>
+<pre><code class="python">from flask import Flask, redirect, request
+
+from hashlib import md5
+
+app = Flask(&quot;url_shortener&quot;)
+
+mapping = {}
+
+@app.route(&quot;/shorten&quot;, methods=[&quot;POST&quot;])
+def shorten():
+    global mapping
+    payload = request.json
+
+    if &quot;url&quot; not in payload:
+        return &quot;Missing URL Parameter&quot;, 400
+
+    # TODO: check if URL is valid
+
+    hash_ = md5()
+    hash_.update(payload[&quot;url&quot;].encode())
+    digest = hash_.hexdigest()[:5]  # limiting to 5 chars. Less the limit more the chances of collission
+
+    if digest not in mapping:
+        mapping[digest] = payload[&quot;url&quot;]
+        return f&quot;Shortened: r/{digest}\n&quot;
+    else:
+        # TODO: check for hash collission
+        return f&quot;Already exists: r/{digest}\n&quot;
+
+
+@app.route(&quot;/r/&lt;hash_&gt;&quot;)
+def redirect_(hash_):
+    if hash_ not in mapping:
+        return &quot;URL Not Found&quot;, 404
+    return redirect(mapping[hash_])
+
+
+if __name__ == &quot;__main__&quot;:
+    app.run(debug=True)
+
+&quot;&quot;&quot;
+OUTPUT:
+
+
+===&gt; SHORTENING
+
+spatel1-mn1:tmp spatel1$ curl localhost:5000/shorten -H &quot;content-type: application/json&quot; --data '{&quot;url&quot;:&quot;https://linkedin.com&quot;}'
+Shortened: r/a62a4
+
+
+===&gt; REDIRECTING, notice the response code 302 and the location header
+
+spatel1-mn1:tmp spatel1$ curl localhost:5000/r/a62a4 -v
+* Uses proxy env variable NO_PROXY == '127.0.0.1'
+*   Trying ::1...
+* TCP_NODELAY set
+* Connection failed
+* connect to ::1 port 5000 failed: Connection refused
+*   Trying 127.0.0.1...
+* TCP_NODELAY set
+* Connected to localhost (127.0.0.1) port 5000 (#0)
+&gt; GET /r/a62a4 HTTP/1.1
+&gt; Host: localhost:5000
+&gt; User-Agent: curl/7.64.1
+&gt; Accept: */*
+&gt;
+* HTTP 1.0, assume close after body
+&lt; HTTP/1.0 302 FOUND
+&lt; Content-Type: text/html; charset=utf-8
+&lt; Content-Length: 247
+&lt; Location: https://linkedin.com
+&lt; Server: Werkzeug/0.15.4 Python/3.7.7
+&lt; Date: Tue, 27 Oct 2020 09:37:12 GMT
+&lt;
+&lt;!DOCTYPE HTML PUBLIC &quot;-//W3C//DTD HTML 3.2 Final//EN&quot;&gt;
+&lt;title&gt;Redirecting...&lt;/title&gt;
+&lt;h1&gt;Redirecting...&lt;/h1&gt;
+* Closing connection 0
+&lt;p&gt;You should be redirected automatically to target URL: &lt;a href=&quot;https://linkedin.com&quot;&gt;https://linkedin.com&lt;/a&gt;.  If not click the link.
+&quot;&quot;&quot;
+</code></pre></div>
+        </div>
+
+        <footer class="col-md-12">
+            <hr>
+            <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
+        </footer>
+        <script>
+            var base_url = "../..",
+                shortcuts = {"search": 83, "next": 78, "help": 191, "previous": 80};
+        </script>
+        <script src="../../js/base.js" defer></script>
+        <script src="../../search/main.js" defer></script>
+
+        <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="Search Modal" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+                <h4 class="modal-title" id="exampleModalLabel">Search</h4>
+            </div>
+            <div class="modal-body">
+                <p>
+                    From here you can search these documents. Enter
+                    your search terms below.
+                </p>
+                <form role="form">
+                    <div class="form-group">
+                        <input type="text" class="form-control" placeholder="Search..." id="mkdocs-search-query" title="Type search term here">
+                    </div>
+                </form>
+                <div id="mkdocs-search-results"></div>
+            </div>
+            <div class="modal-footer">
+            </div>
+        </div>
+    </div>
+</div><div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="Keyboard Shortcuts Modal" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+                <h4 class="modal-title" id="exampleModalLabel">Keyboard Shortcuts</h4>
+            </div>
+            <div class="modal-body">
+              <table class="table">
+                <thead>
+                  <tr>
+                    <th style="width: 20%;">Keys</th>
+                    <th>Action</th>
+                  </tr>
+                </thead>
+                <tbody>
+                  <tr>
+                    <td class="help shortcut"><kbd>?</kbd></td>
+                    <td>Open this help</td>
+                  </tr>
+                  <tr>
+                    <td class="next shortcut"><kbd>n</kbd></td>
+                    <td>Next page</td>
+                  </tr>
+                  <tr>
+                    <td class="prev shortcut"><kbd>p</kbd></td>
+                    <td>Previous page</td>
+                  </tr>
+                  <tr>
+                    <td class="search shortcut"><kbd>s</kbd></td>
+                    <td>Search</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+            <div class="modal-footer">
+            </div>
+        </div>
+    </div>
+</div>
+
+    </body>
+</html>
diff --git a/search/search_index.json b/search/search_index.json
index 4f95998..c59e0b0 100644
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"Hello, World!!!","title":"Home"},{"location":"systems_design/availability/","text":"HA - Availability - Common \u201cNines\u201d Availability is generally expressed as \u201cNines\u201d, common \u2018Nines\u2019 are listed below. Availability % Downtime per year Downtime per month Downtime per week Downtime per day 99%(Two Nines) 3.65 days 7.31 hours 1.68 hours 14.40 minutes 99.5%(Two and a half Nines) 1.83 days 3.65 hours 50.40 minutes 7.20 minutes 99.9%(Three Nines) 8.77 hours 43.83 minutes 10.08 minutes 1.44 minutes 99.95%(Three and a half Nines) 4.38 hours 21.92 minutes 5.04 minutes 43.20 seconds 99.99%(Four Nines) 52.60 minutes 4.38 minutes 1.01 minutes 8.64 seconds 99.995%(Four and a half Nines) 26.30 minutes 2.19 minutes 30.24 seconds 4.32 seconds 99.999%(Five Nines) 5.26 minutes 26.30 seconds 6.05 seconds 864.0 ms Refer https://en.wikipedia.org/wiki/High_availability#Percentage_calculation HA - Availability Serial Components A System with components is operating in the series If failure of a part leads to the combination becoming inoperable. For example if LB in our architecture fails, all access to app tiers will fail. LB and app tiers are connected serially. The combined availability of the system is the product of individual components availability A = Ax x Ay x \u2026.. Refer http://www.eventhelix.com/RealtimeMantra/FaultHandling/system_reliability_availability.htm HA - Availability Parallel Components A System with components is operating in parallel If failure of a part leads to the other part taking over the operations of the failed part. If we have more than one LB and if rest of the LBs can take over the traffic during one LB failure then LBs are operating in parallel The combined availability of the system is A = 1 - ( (1-Ax) x (1-Ax) x \u2026.. ) Refer http://www.eventhelix.com/RealtimeMantra/FaultHandling/system_reliability_availability.htm HA - Core Principles Elimination of single points of failure (SPOF) This means adding redundancy to the system so that the failure of a component does not mean failure of the entire system. Reliable crossover In redundant systems, the crossover point itself tends to become a single point of failure. Reliable systems must provide for reliable crossover. Detection of failures as they occur If the two principles above are observed, then a user may never see a failure Refer https://en.wikipedia.org/wiki/High_availability#Principles HA - SPOF WHAT: Never implement and always eliminate single points of failure. WHEN TO USE: During architecture reviews and new designs. HOW TO USE: Identify single instances on architectural diagrams. Strive for active/active configurations. At the very least we should have a standby to take control when active instances fail. WHY: Maximize availability through multiple instances. KEY TAKEAWAYS: Strive for active/active rather than active/passive solutions. Use load balancers to balance traffic across instances of a service. Use control services with active/passive instances for patterns that require singletons. HA - Reliable Crossover WHAT: Ensure when system components failover they do so reliably. WHEN TO USE: During architecture reviews, failure modeling, and designs. HOW TO USE: Identify how available a system is during the crossover and ensure it is within acceptable limits. WHY: Maximize availability and ensure data handling semantics are preserved. KEY TAKEAWAYS: Strive for active/active rather than active/passive solutions, they have a lesser risk of cross over being unreliable. Use LB and right load balancing methods to ensure reliable failover. Model and build your data systems to ensure data is correctly handled when crossover happens. Generally DB systems follow active/passive semantics for writes. Masters accept writes and when master goes down, follower is promoted to master(active from being passive) to accept writes. We have to be careful here that the cutover never introduces more than one masters. This problem is called a split brain. SRE Use cases SRE works on deciding an acceptable SLA and make sure system is available to achieve the SLA SRE is involved in architecture design right from building the data center to make sure site is not affected by network switch, hardware, power or software failures SRE also run mock drills of failures to see how the system behaves in uncharted territory and comes up with a plan to improve availability if there are misses. https://engineering.linkedin.com/blog/2017/11/resilience-engineering-at-linkedin-with-project-waterbear Post our understanding about HA, our architecture diagram looks something like this below","title":"Availability"},{"location":"systems_design/availability/#ha-availability-common-nines","text":"Availability is generally expressed as \u201cNines\u201d, common \u2018Nines\u2019 are listed below. Availability % Downtime per year Downtime per month Downtime per week Downtime per day 99%(Two Nines) 3.65 days 7.31 hours 1.68 hours 14.40 minutes 99.5%(Two and a half Nines) 1.83 days 3.65 hours 50.40 minutes 7.20 minutes 99.9%(Three Nines) 8.77 hours 43.83 minutes 10.08 minutes 1.44 minutes 99.95%(Three and a half Nines) 4.38 hours 21.92 minutes 5.04 minutes 43.20 seconds 99.99%(Four Nines) 52.60 minutes 4.38 minutes 1.01 minutes 8.64 seconds 99.995%(Four and a half Nines) 26.30 minutes 2.19 minutes 30.24 seconds 4.32 seconds 99.999%(Five Nines) 5.26 minutes 26.30 seconds 6.05 seconds 864.0 ms","title":"HA - Availability - Common \u201cNines\u201d"},{"location":"systems_design/availability/#refer","text":"https://en.wikipedia.org/wiki/High_availability#Percentage_calculation","title":"Refer"},{"location":"systems_design/availability/#ha-availability-serial-components","text":"A System with components is operating in the series If failure of a part leads to the combination becoming inoperable. For example if LB in our architecture fails, all access to app tiers will fail. LB and app tiers are connected serially. The combined availability of the system is the product of individual components availability A = Ax x Ay x \u2026..","title":"HA - Availability Serial Components"},{"location":"systems_design/availability/#refer_1","text":"http://www.eventhelix.com/RealtimeMantra/FaultHandling/system_reliability_availability.htm","title":"Refer"},{"location":"systems_design/availability/#ha-availability-parallel-components","text":"A System with components is operating in parallel If failure of a part leads to the other part taking over the operations of the failed part. If we have more than one LB and if rest of the LBs can take over the traffic during one LB failure then LBs are operating in parallel The combined availability of the system is A = 1 - ( (1-Ax) x (1-Ax) x \u2026.. )","title":"HA - Availability Parallel Components"},{"location":"systems_design/availability/#refer_2","text":"http://www.eventhelix.com/RealtimeMantra/FaultHandling/system_reliability_availability.htm","title":"Refer"},{"location":"systems_design/availability/#ha-core-principles","text":"Elimination of single points of failure (SPOF) This means adding redundancy to the system so that the failure of a component does not mean failure of the entire system. Reliable crossover In redundant systems, the crossover point itself tends to become a single point of failure. Reliable systems must provide for reliable crossover. Detection of failures as they occur If the two principles above are observed, then a user may never see a failure","title":"HA - Core Principles"},{"location":"systems_design/availability/#refer_3","text":"https://en.wikipedia.org/wiki/High_availability#Principles","title":"Refer"},{"location":"systems_design/availability/#ha-spof","text":"WHAT: Never implement and always eliminate single points of failure. WHEN TO USE: During architecture reviews and new designs. HOW TO USE: Identify single instances on architectural diagrams. Strive for active/active configurations. At the very least we should have a standby to take control when active instances fail. WHY: Maximize availability through multiple instances. KEY TAKEAWAYS: Strive for active/active rather than active/passive solutions. Use load balancers to balance traffic across instances of a service. Use control services with active/passive instances for patterns that require singletons.","title":"HA - SPOF"},{"location":"systems_design/availability/#ha-reliable-crossover","text":"WHAT: Ensure when system components failover they do so reliably. WHEN TO USE: During architecture reviews, failure modeling, and designs. HOW TO USE: Identify how available a system is during the crossover and ensure it is within acceptable limits. WHY: Maximize availability and ensure data handling semantics are preserved. KEY TAKEAWAYS: Strive for active/active rather than active/passive solutions, they have a lesser risk of cross over being unreliable. Use LB and right load balancing methods to ensure reliable failover. Model and build your data systems to ensure data is correctly handled when crossover happens. Generally DB systems follow active/passive semantics for writes. Masters accept writes and when master goes down, follower is promoted to master(active from being passive) to accept writes. We have to be careful here that the cutover never introduces more than one masters. This problem is called a split brain.","title":"HA - Reliable Crossover"},{"location":"systems_design/availability/#sre-use-cases","text":"SRE works on deciding an acceptable SLA and make sure system is available to achieve the SLA SRE is involved in architecture design right from building the data center to make sure site is not affected by network switch, hardware, power or software failures SRE also run mock drills of failures to see how the system behaves in uncharted territory and comes up with a plan to improve availability if there are misses. https://engineering.linkedin.com/blog/2017/11/resilience-engineering-at-linkedin-with-project-waterbear Post our understanding about HA, our architecture diagram looks something like this below","title":"SRE Use cases"},{"location":"systems_design/conclusion/","text":"Conclusion Armed with these principles, we hope the course will give a fresh perspective to design software systems. It might be over engineering to get all this on day zero. But some are really important from day 0 like eliminating single points of failure, making scalable services by just increasing replicas. As a bottleneck is reached, we can split code by services, shard data to scale. As the organisation matures, bringing in chaos engineering to measure how systems react to failure will help in designing robust software systems.","title":"Conclusion"},{"location":"systems_design/conclusion/#conclusion","text":"Armed with these principles, we hope the course will give a fresh perspective to design software systems. It might be over engineering to get all this on day zero. But some are really important from day 0 like eliminating single points of failure, making scalable services by just increasing replicas. As a bottleneck is reached, we can split code by services, shard data to scale. As the organisation matures, bringing in chaos engineering to measure how systems react to failure will help in designing robust software systems.","title":"Conclusion"},{"location":"systems_design/fault-tolerance/","text":"Fault Tolerance Failures are not avoidable in any system and will happen all the time, hence we need to build systems that can tolerate failures or recover from them. In systems, failure is the norm rather than the exception. \"Anything that can go wrong will go wrong\u201d -- Murphy\u2019s Law \u201cComplex systems contain changing mixtures of failures latent within them\u201d -- How Complex Systems Fail. Fault Tolerance - Failure Metrics Common failure metrics that get measured and tracked for any system. Mean time to repair (MTTR): The average time to repair and restore a failed system. Mean time between failures (MTBF): The average operational time between one device failure or system breakdown and the next. Mean time to failure (MTTF): The average time a device or system is expected to function before it fails. Mean time to detect (MTTD): The average time between the onset of a problem and when the organization detects it. Mean time to investigate (MTTI): The average time between the detection of an incident and when the organization begins to investigate its cause and solution. Mean time to restore service (MTRS): The average elapsed time from the detection of an incident until the affected system or component is again available to users. Mean time between system incidents (MTBSI): The average elapsed time between the detection of two consecutive incidents. MTBSI can be calculated by adding MTBF and MTRS (MTBSI = MTBF + MTRS). Failure rate: Another reliability metric, which measures the frequency with which a component or system fails. It is expressed as a number of failures over a unit of time. Refer https://www.splunk.com/en_us/data-insider/what-is-mean-time-to-repair.html Fault Tolerance - Fault Isolation Terms Systems should have a short circuit. Say in our content sharing system, if \u201cNotifications\u201d is not working, the site should gracefully handle that failure by removing the functionality instead of taking the whole site down. Swimlane is one of the commonly used fault isolation methodology. Swimlane adds a barrier to the service from other services so that failure on either of them won\u2019t affect the other. Say we roll out a new feature \u2018Advertisement\u2019 in our content sharing app. We can have two architectures If Ads are generated on the fly synchronously during each Newsfeed request, the faults in Ads feature gets propagated to Newsfeed feature. Instead if we swimlane \u201cGeneration of Ads\u201d service and use a shared storage to populate Newsfeed App, Ads failures won\u2019t cascade to Newsfeed and worst case if Ads don\u2019t meet SLA , we can have Newsfeed without Ads. Let's take another example, we come up with a new model for our Content sharing App. Here we roll out enterprise content sharing App where enterprises pay for the service and the content should never be shared outside the enterprise. Swimlane Principles Principle 1: Nothing is shared (also known as \u201cshare as little as possible\u201d). The less that is shared within a swim lane, the more fault isolative the swim lane becomes. (as shown in Enterprise usecase) Principle 2: Nothing crosses a swim lane boundary. Synchronous (defined by expecting a request\u2014not the transfer protocol) communication never crosses a swim lane boundary; if it does, the boundary is drawn incorrectly. (as shown in Ads feature) Swimlane Approaches Approach 1: Swim lane the money-maker. Never allow your cash register to be compromised by other systems. (Tier 1 vs Tier 2 in enterprise use case) Approach 2: Swim lane the biggest sources of incidents. Identify the recurring causes of pain and isolate them.(if Ads feature is in code yellow, swim laning it is the best option) Approach 3: Swim lane natural barriers. Customer boundaries make good swim lanes.(Public vs Enterprise customers) Refer https://learning.oreilly.com/library/view/the-art-of/9780134031408/ch21.html#ch21 SRE Use cases: Work with the DC tech or cloud team to distribute infrastructure such that its immune to switch or power failures by creating fault zones within a Data Center https://docs.microsoft.com/en-us/azure/virtual-machines/manage-availability#use-availability-zones-to-protect-from-datacenter-level-failures Work with the partners and design interaction between services such that one service breakdown is not amplified in a cascading fashion to all upstreams","title":"Fault Tolerance"},{"location":"systems_design/fault-tolerance/#fault-tolerance","text":"Failures are not avoidable in any system and will happen all the time, hence we need to build systems that can tolerate failures or recover from them. In systems, failure is the norm rather than the exception. \"Anything that can go wrong will go wrong\u201d -- Murphy\u2019s Law \u201cComplex systems contain changing mixtures of failures latent within them\u201d -- How Complex Systems Fail.","title":"Fault Tolerance"},{"location":"systems_design/fault-tolerance/#fault-tolerance-failure-metrics","text":"Common failure metrics that get measured and tracked for any system. Mean time to repair (MTTR): The average time to repair and restore a failed system. Mean time between failures (MTBF): The average operational time between one device failure or system breakdown and the next. Mean time to failure (MTTF): The average time a device or system is expected to function before it fails. Mean time to detect (MTTD): The average time between the onset of a problem and when the organization detects it. Mean time to investigate (MTTI): The average time between the detection of an incident and when the organization begins to investigate its cause and solution. Mean time to restore service (MTRS): The average elapsed time from the detection of an incident until the affected system or component is again available to users. Mean time between system incidents (MTBSI): The average elapsed time between the detection of two consecutive incidents. MTBSI can be calculated by adding MTBF and MTRS (MTBSI = MTBF + MTRS). Failure rate: Another reliability metric, which measures the frequency with which a component or system fails. It is expressed as a number of failures over a unit of time.","title":"Fault Tolerance - Failure Metrics"},{"location":"systems_design/fault-tolerance/#refer","text":"https://www.splunk.com/en_us/data-insider/what-is-mean-time-to-repair.html","title":"Refer"},{"location":"systems_design/fault-tolerance/#fault-tolerance-fault-isolation-terms","text":"Systems should have a short circuit. Say in our content sharing system, if \u201cNotifications\u201d is not working, the site should gracefully handle that failure by removing the functionality instead of taking the whole site down. Swimlane is one of the commonly used fault isolation methodology. Swimlane adds a barrier to the service from other services so that failure on either of them won\u2019t affect the other. Say we roll out a new feature \u2018Advertisement\u2019 in our content sharing app. We can have two architectures If Ads are generated on the fly synchronously during each Newsfeed request, the faults in Ads feature gets propagated to Newsfeed feature. Instead if we swimlane \u201cGeneration of Ads\u201d service and use a shared storage to populate Newsfeed App, Ads failures won\u2019t cascade to Newsfeed and worst case if Ads don\u2019t meet SLA , we can have Newsfeed without Ads. Let's take another example, we come up with a new model for our Content sharing App. Here we roll out enterprise content sharing App where enterprises pay for the service and the content should never be shared outside the enterprise.","title":"Fault Tolerance - Fault Isolation Terms"},{"location":"systems_design/fault-tolerance/#swimlane-principles","text":"Principle 1: Nothing is shared (also known as \u201cshare as little as possible\u201d). The less that is shared within a swim lane, the more fault isolative the swim lane becomes. (as shown in Enterprise usecase) Principle 2: Nothing crosses a swim lane boundary. Synchronous (defined by expecting a request\u2014not the transfer protocol) communication never crosses a swim lane boundary; if it does, the boundary is drawn incorrectly. (as shown in Ads feature)","title":"Swimlane Principles"},{"location":"systems_design/fault-tolerance/#swimlane-approaches","text":"Approach 1: Swim lane the money-maker. Never allow your cash register to be compromised by other systems. (Tier 1 vs Tier 2 in enterprise use case) Approach 2: Swim lane the biggest sources of incidents. Identify the recurring causes of pain and isolate them.(if Ads feature is in code yellow, swim laning it is the best option) Approach 3: Swim lane natural barriers. Customer boundaries make good swim lanes.(Public vs Enterprise customers)","title":"Swimlane Approaches"},{"location":"systems_design/fault-tolerance/#refer_1","text":"https://learning.oreilly.com/library/view/the-art-of/9780134031408/ch21.html#ch21","title":"Refer"},{"location":"systems_design/fault-tolerance/#sre-use-cases","text":"Work with the DC tech or cloud team to distribute infrastructure such that its immune to switch or power failures by creating fault zones within a Data Center https://docs.microsoft.com/en-us/azure/virtual-machines/manage-availability#use-availability-zones-to-protect-from-datacenter-level-failures Work with the partners and design interaction between services such that one service breakdown is not amplified in a cascading fashion to all upstreams","title":"SRE Use cases:"},{"location":"systems_design/intro/","text":"Systems Design Pre - Requisites Fundamentals of common software system components: - Operating Systems - Networking - Databases RDBMS/NoSQL What to expect from this training Thinking about and designing for scalability, availability, and reliability of large scale software systems. What is not covered under this training Individual software components\u2019 scalability and reliability concerns like e.g. Databases, while the same scalability principles and thinking can be applied, these individual components have their own specific nuances when scaling them and thinking about their reliability. More light will be shed on concepts rather than on setting up and configuring components like Loadbalancers to achieve scalability, availability and reliability of systems Training Content Introduction Scalability High Availability Fault Tolerance Introduction So, how do you go about learning to design a system? \u201d Like most great questions, it showed a level of naivety that was breathtaking. The only short answer I could give was, essentially, that you learned how to design a system by designing systems and finding out what works and what doesn\u2019t work.\u201d Jim Waldo, Sun Microsystems, On System Design As software and hardware systems have multiple moving parts, we need to think about how those parts will grow, their failure modes, their inter-dependencies, how it will impact the users and the business. There is no one-shot method or way to learn or do system design, we only learn to design systems by designing and iterating on them. This course will be a starter to make one think about scalability, availability, and fault tolerance during systems design. Backstory Let\u2019s design a simple content sharing application where users can share photos, media in our application which can be liked by their friends. Let\u2019s start with a simple design of the application and evolve it as we learn system design concepts","title":"Intro"},{"location":"systems_design/intro/#systems-design","text":"","title":"Systems Design"},{"location":"systems_design/intro/#pre-requisites","text":"Fundamentals of common software system components: - Operating Systems - Networking - Databases RDBMS/NoSQL","title":"Pre - Requisites"},{"location":"systems_design/intro/#what-to-expect-from-this-training","text":"Thinking about and designing for scalability, availability, and reliability of large scale software systems.","title":"What to expect from this training"},{"location":"systems_design/intro/#what-is-not-covered-under-this-training","text":"Individual software components\u2019 scalability and reliability concerns like e.g. Databases, while the same scalability principles and thinking can be applied, these individual components have their own specific nuances when scaling them and thinking about their reliability. More light will be shed on concepts rather than on setting up and configuring components like Loadbalancers to achieve scalability, availability and reliability of systems","title":"What is not covered under this training"},{"location":"systems_design/intro/#training-content","text":"Introduction Scalability High Availability Fault Tolerance","title":"Training Content"},{"location":"systems_design/intro/#introduction","text":"So, how do you go about learning to design a system? \u201d Like most great questions, it showed a level of naivety that was breathtaking. The only short answer I could give was, essentially, that you learned how to design a system by designing systems and finding out what works and what doesn\u2019t work.\u201d Jim Waldo, Sun Microsystems, On System Design As software and hardware systems have multiple moving parts, we need to think about how those parts will grow, their failure modes, their inter-dependencies, how it will impact the users and the business. There is no one-shot method or way to learn or do system design, we only learn to design systems by designing and iterating on them. This course will be a starter to make one think about scalability, availability, and fault tolerance during systems design.","title":"Introduction"},{"location":"systems_design/intro/#backstory","text":"Let\u2019s design a simple content sharing application where users can share photos, media in our application which can be liked by their friends. Let\u2019s start with a simple design of the application and evolve it as we learn system design concepts","title":"Backstory"},{"location":"systems_design/scalability/","text":"Scalability What does scalability mean for a system/service? A system is composed of services/components, each service/component scalability needs to be tackled separately, and the scalability of the system as a whole. A service is said to be scalable if, as resources are added to the system, it results in increased performance in a manner proportional to resources added An always-on service is said to be scalable if adding resources to facilitate redundancy does not result in a loss of performance Refer https://www.allthingsdistributed.com/2006/03/a_word_on_scalability.html Scalability - AKF Scale Cube The Scale Cube is a model for segmenting services, defining microservices, and scaling products. It also creates a common language for teams to discuss scale related options in designing solutions. Following section talks about certain scaling patterns based on our inferences from AKF cube Scalability - Horizontal scaling Horizontal scaling stands for cloning of an application or service such that work can easily be distributed across instances with absolutely no bias. Lets see how our monolithic application improves with this principle Here DB is scaled separately from the application. This is to let you know each component\u2019s scaling capabilities can be different. Usually web applications can be scaled by adding resources unless there is no state stored inside the application. But DBs can be scaled only for Reads by adding more followers but Writes have to go to only one master to make sure data is consistent. There are some DBs which support multi master writes but we are keeping them out of scope at this point. Apps should be able to differentiate between Read and Writes to choose appropriate DB servers. Load balancers can split traffic between identical servers transparently. WHAT: Duplication of services or databases to spread transaction load. WHEN TO USE: Databases with a very high read-to-write ratio (5:1 or greater\u2014the higher the better). Because only read replicas of DBs can be scaled, not the Master. HOW TO USE: Simply clone services and implement a load balancer. For databases, ensure that the accessing code understands the difference between a read and a write. WHY: Allows for fast scale of transactions at the cost of duplicated data and functionality. KEY TAKEAWAYS: This is fast to implement, is low cost from a developer effort perspective, and can scale transaction volumes nicely. However, they tend to be high cost from the perspective of the operational cost of data. Cost here means if we have 3 followers and 1 Master DB, the same database will be stored as 4 copies in the 4 servers. Hence added storage cost Refer https://learning.oreilly.com/library/view/the-art-of/9780134031408/ch23.html Scalability Pattern - Load Balancing Improves the distribution of workloads across multiple computing resources, such as computers, a computer cluster, network links, central processing units, or disk drives. Commonly used technique is load balancing traffic across identical server clusters. Similar philosophy is used to load balance traffic across network links by ECMP , disk drives by RAID etc Aims to optimize resource use, maximize throughput, minimize response time, and avoid overload of any single resource. Using multiple components with load balancing instead of a single component may increase reliability and availability through redundancy. In our updated architecture diagram we have 4 servers to handle app traffic instead of a single server The device or system that performs load balancing is called a load balancer, abbreviated as LB. Refer https://en.wikipedia.org/wiki/Load_balancing_(computing) https://blog.envoyproxy.io/introduction-to-modern-network-load-balancing-and-proxying-a57f6ff80236 https://learning.oreilly.com/library/view/load-balancing-in/9781492038009/ https://learning.oreilly.com/library/view/practical-load-balancing/9781430236801/ http://shop.oreilly.com/product/9780596000509.do Scalability Pattern - LB Tasks What does an LB do? Service discovery: What backends are available in the system? In our architecture, 4 servers are available to serve App traffic. LB acts as a single endpoint that clients can use transparently to reach one of the 4 servers. Health checking: What backends are currently healthy and available to accept requests? If one out of the 4 App servers turns bad, LB should automatically short circuit the path so that clients don\u2019t sense any application downtime Load balancing: What algorithm should be used to balance individual requests across the healthy backends? There are many algorithms to distribute traffic across one of the four servers. Based on observations/experience, SRE can pick the algorithm that suits their pattern Scalability Pattern - LB Methods Common Load Balancing Methods Least Connection Method directs traffic to the server with the fewest active connections. Most useful when there are a large number of persistent connections in the traffic unevenly distributed between the servers. Works if clients maintain long lived connections Least Response Time Method directs traffic to the server with the fewest active connections and the lowest average response time. Here response time is used to provide feedback of server\u2019s health Round Robin Method rotates servers by directing traffic to the first available server and then moves that server to the bottom of the queue. Most useful when servers are of equal specification and there are not many persistent connections. IP Hash the IP address of the client determines which server receives the request. This can sometimes cause skewness in distribution but is useful if apps store some state locally and need some stickiness More advanced client/server-side example techniques - https://docs.nginx.com/nginx/admin-guide/load-balancer/ - http://cbonte.github.io/haproxy-dconv/2.2/intro.html#3.3.5 - https://twitter.github.io/finagle/guide/Clients.html#load-balancing Scalability Pattern - Caching - Content Delivery Networks (CDN) CDNs are added closer to the client\u2019s location. If the app has static data like images, Javascript, CSS which don\u2019t change very often, they can be cached. Since our example is a content sharing site, static content can be cached in CDNs with a suitable expiry. WHAT: Use CDNs (content delivery networks) to offload traffic from your site. WHEN TO USE: When speed improvements and scale warrant the additional cost. HOW TO USE: Most CDNs leverage DNS to serve content on your site\u2019s behalf. Thus you may need to make minor DNS changes or additions and move content to be served from new subdomains. Eg media-exp1.licdn.com is a domain used by Linkedin to serve static content Here a CNAME points the domain to the DNS of CDN provider dig media-exp1.licdn.com +short 2-01-2c3e-005c.cdx.cedexis.net. WHY: CDNs help offload traffic spikes and are often economical ways to scale parts of a site\u2019s traffic. They also often substantially improve page download times. KEY TAKEAWAYS: CDNs are a fast and simple way to offset the spikiness of traffic as well as traffic growth in general. Make sure you perform a cost-benefit analysis and monitor the CDN usage. If CDNs have a lot of cache misses, then we don\u2019t gain much from CDN and are still serving requests using our compute resources. Scalability - Microservices This pattern represents the separation of work by service or function within the application. Microservices are meant to address the issues associated with growth and complexity in the code base and data sets. The intent is to create fault isolation as well as to reduce response times. Microservices can scale transactions, data sizes, and codebase sizes. They are most effective in scaling the size and complexity of your codebase. They tend to cost a bit more than horizontal scaling because the engineering team needs to rewrite services or, at the very least, disaggregate them from the original monolithic application. WHAT: Sometimes referred to as scale through services or resources, this rule focuses on scaling by splitting data sets, transactions, and engineering teams along verb (services) or noun (resources) boundaries. WHEN TO USE: Very large data sets where relations between data are not necessary. Large, complex systems where scaling engineering resources requires specialization. HOW TO USE: Split up actions by using verbs, or resources by using nouns, or use a mix. Split both the services and the data along the lines defined by the verb/noun approach. WHY: Allows for efficient scaling of not only transactions but also very large data sets associated with those transactions. It also allows for the efficient scaling of teams. KEY TAKEAWAYS: Microservices allow for efficient scaling of transactions, large data sets, and can help with fault isolation. It helps reduce the communication overhead of teams. The codebase becomes less complex as disjoint features are decoupled and spun as new services thereby letting each service scale independently specific to its requirement. Refer https://learning.oreilly.com/library/view/the-art-of/9780134031408/ch23.html Scalability - Sharding This pattern represents the separation of work based on attributes that are looked up or determined at the time of the transaction. Most often, these are implemented as splits by requestor, customer, or client. Very often, a lookup service or deterministic algorithm will need to be written for these types of splits. Sharding aids in scaling transaction growth, scaling instruction sets, and decreasing processing time (the last by limiting the data necessary to perform any transaction). This is more effective at scaling growth in customers or clients. It can aid with disaster recovery efforts, and limit the impact of incidents to only a specific segment of customers. Here the auth data is sharded based on user names so that DBs can respond faster as the amount of data DBs have to work on has drastically reduced during queries. There can be other ways to split Here the whole data centre is split and replicated and clients are directed to a data centre based on their geography. This helps in improving performance as clients are directed to the closest Data centre and performance increases as we add more data centres. There are some replication and consistency overhead with this approach one needs to be aware of. This also gives fault tolerance by rolling out test features to one site and rollback if there is an impact to that geography WHAT: This is very often a split by some unique aspect of the customer such as customer ID, name, geography, and so on. WHEN TO USE: Very large, similar data sets such as large and rapidly growing customer bases or when the response time for a geographically distributed customer base is important. HOW TO USE: Identify something you know about the customer, such as customer ID, last name, geography, or device, and split or partition both data and services based on that attribute. WHY: Rapid customer growth exceeds other forms of data growth, or you have the need to perform fault isolation between certain customer groups as you scale. KEY TAKEAWAYS: Shards are effective at helping you to scale customer bases but can also be applied to other very large data sets that can\u2019t be pulled apart using the microservices methodology. Refer https://learning.oreilly.com/library/view/the-art-of/9780134031408/ch23.html SRE Use cases SREs in coordination with the network team work on how to map users traffic to a particular site. https://engineering.linkedin.com/blog/2017/05/trafficshift--load-testing-at-scale SREs work closely with the Dev team to split monoliths to multiple microservices that are easy to run and manage SREs work on improving Load Balancers' reliability, service discovery and performance SREs work closely to split Data into shards and manage data integrity and consistency. https://engineering.linkedin.com/espresso/introducing-espresso-linkedins-hot-new-distributed-document-store SREs work to set up, configure and improve CDN cache hit rate.","title":"Scalability"},{"location":"systems_design/scalability/#scalability","text":"What does scalability mean for a system/service? A system is composed of services/components, each service/component scalability needs to be tackled separately, and the scalability of the system as a whole. A service is said to be scalable if, as resources are added to the system, it results in increased performance in a manner proportional to resources added An always-on service is said to be scalable if adding resources to facilitate redundancy does not result in a loss of performance","title":"Scalability"},{"location":"systems_design/scalability/#refer","text":"https://www.allthingsdistributed.com/2006/03/a_word_on_scalability.html","title":"Refer"},{"location":"systems_design/scalability/#scalability-akf-scale-cube","text":"The Scale Cube is a model for segmenting services, defining microservices, and scaling products. It also creates a common language for teams to discuss scale related options in designing solutions. Following section talks about certain scaling patterns based on our inferences from AKF cube","title":"Scalability - AKF Scale Cube"},{"location":"systems_design/scalability/#scalability-horizontal-scaling","text":"Horizontal scaling stands for cloning of an application or service such that work can easily be distributed across instances with absolutely no bias. Lets see how our monolithic application improves with this principle Here DB is scaled separately from the application. This is to let you know each component\u2019s scaling capabilities can be different. Usually web applications can be scaled by adding resources unless there is no state stored inside the application. But DBs can be scaled only for Reads by adding more followers but Writes have to go to only one master to make sure data is consistent. There are some DBs which support multi master writes but we are keeping them out of scope at this point. Apps should be able to differentiate between Read and Writes to choose appropriate DB servers. Load balancers can split traffic between identical servers transparently. WHAT: Duplication of services or databases to spread transaction load. WHEN TO USE: Databases with a very high read-to-write ratio (5:1 or greater\u2014the higher the better). Because only read replicas of DBs can be scaled, not the Master. HOW TO USE: Simply clone services and implement a load balancer. For databases, ensure that the accessing code understands the difference between a read and a write. WHY: Allows for fast scale of transactions at the cost of duplicated data and functionality. KEY TAKEAWAYS: This is fast to implement, is low cost from a developer effort perspective, and can scale transaction volumes nicely. However, they tend to be high cost from the perspective of the operational cost of data. Cost here means if we have 3 followers and 1 Master DB, the same database will be stored as 4 copies in the 4 servers. Hence added storage cost","title":"Scalability - Horizontal scaling"},{"location":"systems_design/scalability/#refer_1","text":"https://learning.oreilly.com/library/view/the-art-of/9780134031408/ch23.html","title":"Refer"},{"location":"systems_design/scalability/#scalability-pattern-load-balancing","text":"Improves the distribution of workloads across multiple computing resources, such as computers, a computer cluster, network links, central processing units, or disk drives. Commonly used technique is load balancing traffic across identical server clusters. Similar philosophy is used to load balance traffic across network links by ECMP , disk drives by RAID etc Aims to optimize resource use, maximize throughput, minimize response time, and avoid overload of any single resource. Using multiple components with load balancing instead of a single component may increase reliability and availability through redundancy. In our updated architecture diagram we have 4 servers to handle app traffic instead of a single server The device or system that performs load balancing is called a load balancer, abbreviated as LB.","title":"Scalability Pattern - Load Balancing"},{"location":"systems_design/scalability/#refer_2","text":"https://en.wikipedia.org/wiki/Load_balancing_(computing) https://blog.envoyproxy.io/introduction-to-modern-network-load-balancing-and-proxying-a57f6ff80236 https://learning.oreilly.com/library/view/load-balancing-in/9781492038009/ https://learning.oreilly.com/library/view/practical-load-balancing/9781430236801/ http://shop.oreilly.com/product/9780596000509.do","title":"Refer"},{"location":"systems_design/scalability/#scalability-pattern-lb-tasks","text":"What does an LB do?","title":"Scalability Pattern - LB Tasks"},{"location":"systems_design/scalability/#service-discovery","text":"What backends are available in the system? In our architecture, 4 servers are available to serve App traffic. LB acts as a single endpoint that clients can use transparently to reach one of the 4 servers.","title":"Service discovery:"},{"location":"systems_design/scalability/#health-checking","text":"What backends are currently healthy and available to accept requests? If one out of the 4 App servers turns bad, LB should automatically short circuit the path so that clients don\u2019t sense any application downtime","title":"Health checking:"},{"location":"systems_design/scalability/#load-balancing","text":"What algorithm should be used to balance individual requests across the healthy backends? There are many algorithms to distribute traffic across one of the four servers. Based on observations/experience, SRE can pick the algorithm that suits their pattern","title":"Load balancing:"},{"location":"systems_design/scalability/#scalability-pattern-lb-methods","text":"Common Load Balancing Methods","title":"Scalability Pattern - LB Methods"},{"location":"systems_design/scalability/#least-connection-method","text":"directs traffic to the server with the fewest active connections. Most useful when there are a large number of persistent connections in the traffic unevenly distributed between the servers. Works if clients maintain long lived connections","title":"Least Connection Method"},{"location":"systems_design/scalability/#least-response-time-method","text":"directs traffic to the server with the fewest active connections and the lowest average response time. Here response time is used to provide feedback of server\u2019s health","title":"Least Response Time Method"},{"location":"systems_design/scalability/#round-robin-method","text":"rotates servers by directing traffic to the first available server and then moves that server to the bottom of the queue. Most useful when servers are of equal specification and there are not many persistent connections.","title":"Round Robin Method"},{"location":"systems_design/scalability/#ip-hash","text":"the IP address of the client determines which server receives the request. This can sometimes cause skewness in distribution but is useful if apps store some state locally and need some stickiness More advanced client/server-side example techniques - https://docs.nginx.com/nginx/admin-guide/load-balancer/ - http://cbonte.github.io/haproxy-dconv/2.2/intro.html#3.3.5 - https://twitter.github.io/finagle/guide/Clients.html#load-balancing","title":"IP Hash"},{"location":"systems_design/scalability/#scalability-pattern-caching-content-delivery-networks-cdn","text":"CDNs are added closer to the client\u2019s location. If the app has static data like images, Javascript, CSS which don\u2019t change very often, they can be cached. Since our example is a content sharing site, static content can be cached in CDNs with a suitable expiry. WHAT: Use CDNs (content delivery networks) to offload traffic from your site. WHEN TO USE: When speed improvements and scale warrant the additional cost. HOW TO USE: Most CDNs leverage DNS to serve content on your site\u2019s behalf. Thus you may need to make minor DNS changes or additions and move content to be served from new subdomains. Eg media-exp1.licdn.com is a domain used by Linkedin to serve static content Here a CNAME points the domain to the DNS of CDN provider dig media-exp1.licdn.com +short 2-01-2c3e-005c.cdx.cedexis.net. WHY: CDNs help offload traffic spikes and are often economical ways to scale parts of a site\u2019s traffic. They also often substantially improve page download times. KEY TAKEAWAYS: CDNs are a fast and simple way to offset the spikiness of traffic as well as traffic growth in general. Make sure you perform a cost-benefit analysis and monitor the CDN usage. If CDNs have a lot of cache misses, then we don\u2019t gain much from CDN and are still serving requests using our compute resources.","title":"Scalability Pattern - Caching - Content Delivery Networks (CDN)"},{"location":"systems_design/scalability/#scalability-microservices","text":"This pattern represents the separation of work by service or function within the application. Microservices are meant to address the issues associated with growth and complexity in the code base and data sets. The intent is to create fault isolation as well as to reduce response times. Microservices can scale transactions, data sizes, and codebase sizes. They are most effective in scaling the size and complexity of your codebase. They tend to cost a bit more than horizontal scaling because the engineering team needs to rewrite services or, at the very least, disaggregate them from the original monolithic application. WHAT: Sometimes referred to as scale through services or resources, this rule focuses on scaling by splitting data sets, transactions, and engineering teams along verb (services) or noun (resources) boundaries. WHEN TO USE: Very large data sets where relations between data are not necessary. Large, complex systems where scaling engineering resources requires specialization. HOW TO USE: Split up actions by using verbs, or resources by using nouns, or use a mix. Split both the services and the data along the lines defined by the verb/noun approach. WHY: Allows for efficient scaling of not only transactions but also very large data sets associated with those transactions. It also allows for the efficient scaling of teams. KEY TAKEAWAYS: Microservices allow for efficient scaling of transactions, large data sets, and can help with fault isolation. It helps reduce the communication overhead of teams. The codebase becomes less complex as disjoint features are decoupled and spun as new services thereby letting each service scale independently specific to its requirement.","title":"Scalability - Microservices"},{"location":"systems_design/scalability/#refer_3","text":"https://learning.oreilly.com/library/view/the-art-of/9780134031408/ch23.html","title":"Refer"},{"location":"systems_design/scalability/#scalability-sharding","text":"This pattern represents the separation of work based on attributes that are looked up or determined at the time of the transaction. Most often, these are implemented as splits by requestor, customer, or client. Very often, a lookup service or deterministic algorithm will need to be written for these types of splits. Sharding aids in scaling transaction growth, scaling instruction sets, and decreasing processing time (the last by limiting the data necessary to perform any transaction). This is more effective at scaling growth in customers or clients. It can aid with disaster recovery efforts, and limit the impact of incidents to only a specific segment of customers. Here the auth data is sharded based on user names so that DBs can respond faster as the amount of data DBs have to work on has drastically reduced during queries. There can be other ways to split Here the whole data centre is split and replicated and clients are directed to a data centre based on their geography. This helps in improving performance as clients are directed to the closest Data centre and performance increases as we add more data centres. There are some replication and consistency overhead with this approach one needs to be aware of. This also gives fault tolerance by rolling out test features to one site and rollback if there is an impact to that geography WHAT: This is very often a split by some unique aspect of the customer such as customer ID, name, geography, and so on. WHEN TO USE: Very large, similar data sets such as large and rapidly growing customer bases or when the response time for a geographically distributed customer base is important. HOW TO USE: Identify something you know about the customer, such as customer ID, last name, geography, or device, and split or partition both data and services based on that attribute. WHY: Rapid customer growth exceeds other forms of data growth, or you have the need to perform fault isolation between certain customer groups as you scale. KEY TAKEAWAYS: Shards are effective at helping you to scale customer bases but can also be applied to other very large data sets that can\u2019t be pulled apart using the microservices methodology.","title":"Scalability - Sharding"},{"location":"systems_design/scalability/#refer_4","text":"https://learning.oreilly.com/library/view/the-art-of/9780134031408/ch23.html","title":"Refer"},{"location":"systems_design/scalability/#sre-use-cases","text":"SREs in coordination with the network team work on how to map users traffic to a particular site. https://engineering.linkedin.com/blog/2017/05/trafficshift--load-testing-at-scale SREs work closely with the Dev team to split monoliths to multiple microservices that are easy to run and manage SREs work on improving Load Balancers' reliability, service discovery and performance SREs work closely to split Data into shards and manage data integrity and consistency. https://engineering.linkedin.com/espresso/introducing-espresso-linkedins-hot-new-distributed-document-store SREs work to set up, configure and improve CDN cache hit rate.","title":"SRE Use cases"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"Hello, World!!!","title":"Home"},{"location":"git/branches/","text":"Working With Branches Coming back to our local repo which has two commits. So far, what we have is a single line of history. Commits are chained in a single line. But sometimes you may have a need to work on two different features in parallel in the same repo. Now one option here could be making a new folder/repo with the same code and use that for another feature development. But there's a better way. Use branches. Since git follows tree like structure for commits, we can use branches to work on different sets of features. From a commit, two or more branches can be created and branches can also be merged. Using branches, there can exist multiple lines of histories and we can checkout to any of them and work on it. Checking out, as we discussed earlier, would simply mean replacing contents of the directory (repo) with contents snapshot at the checked out version. Let's create a branch and see how it looks like: spatel1-mn1:school-of-sre spatel1$ git branch b1 spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph * 7f3b00e (HEAD - master, b1) adding file 2 * df2fb7a adding file 1 We create a branch called b1 . Git log tells us that b1 also points to the last commit (7f3b00e) but the HEAD is still pointing to master. If you remember, HEAD points to the commit/reference wherever you are checkout to. So if we checkout to b1 , HEAD should point to that. Let's confirm: spatel1-mn1:school-of-sre spatel1$ git checkout b1 Switched to branch 'b1' spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph * 7f3b00e (HEAD - b1, master) adding file 2 * df2fb7a adding file 1 b1 still points to the same commit but HEAD now points to b1 . Since we create a branch at commit 7f3b00e , there will be two lines of histories starting this commit. Depending on which branch you are checked out on, the line of history will progress. At this moment, we are checked out on branch b1 , so making a new commit will advance branch reference b1 to that commit and current b1 commit will become its parent. Let's do that. # Creating a file and making a commit spatel1-mn1:school-of-sre spatel1$ echo I am a file in b1 branch b1.txt spatel1-mn1:school-of-sre spatel1$ git add b1.txt spatel1-mn1:school-of-sre spatel1$ git commit -m adding b1 file [b1 872a38f] adding b1 file 1 file changed, 1 insertion(+) create mode 100644 b1.txt # The new line of history spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph * 872a38f (HEAD - b1) adding b1 file * 7f3b00e (master) adding file 2 * df2fb7a adding file 1 spatel1-mn1:school-of-sre spatel1$ Do note that master is still pointing to the old commit it was pointing to. We can now checkout to master branch and make commits there. This will result in another line of history starting from commit 7f3b00e. # checkout to master branch spatel1-mn1:school-of-sre spatel1$ git checkout master Switched to branch 'master' # Creating a new commit on master branch spatel1-mn1:school-of-sre spatel1$ echo new file in master branch master.txt spatel1-mn1:school-of-sre spatel1$ git add master.txt spatel1-mn1:school-of-sre spatel1$ git commit -m adding master.txt file [master 60dc441] adding master.txt file 1 file changed, 1 insertion(+) create mode 100644 master.txt # The history line spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph * 60dc441 (HEAD - master) adding master.txt file * 7f3b00e adding file 2 * df2fb7a adding file 1 Notice how branch b1 is not visible here since we are checkout on master. Let's try to visualize both to get the whole picture: spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all * 60dc441 (HEAD - master) adding master.txt file | * 872a38f (b1) adding b1 file |/ * 7f3b00e adding file 2 * df2fb7a adding file 1 Above tree structure should make things clear. Notice a clear branch/fork on commit 7f3b00e. This is how we create branches. Now they both are two separate lines of history on which feature development can be done independently. To reiterate, internally, git is just a tree of commits. Branch names (human readable) are pointers to those commits in the tree. We use various git commands to work with the tree structure and references. Git accordingly modifies contents of our repo. Merges Now say the feature you were working on branch b1 is complete. And you need to merge it on master branch, where all the final version of code goes. So first you will checkout to branch master and then you will pull the latest code from upstream (eg: GitHub). Then you need to merge your code from b1 into master. And there could be two ways this can be done. Here is the current history: spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all * 60dc441 (HEAD - master) adding master.txt file | * 872a38f (b1) adding b1 file |/ * 7f3b00e adding file 2 * df2fb7a adding file 1 Option 1: Directly merge the branch. Merging the branch b1 into master will result in a new merge commit which will merge changes from two different lines of history and create a new commit of the result. spatel1-mn1:school-of-sre spatel1$ git merge b1 Merge made by the 'recursive' strategy. b1.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 b1.txt spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all * 8fc28f9 (HEAD - master) Merge branch 'b1' |\\ | * 872a38f (b1) adding b1 file * | 60dc441 adding master.txt file |/ * 7f3b00e adding file 2 * df2fb7a adding file 1 You can see a new merge commit created (8fc28f9). You will be prompted for the commit message. If there are a lot of branches in the repo, this result will end-up with a lot of merge commits. Which looks ugly compared to a single line of history of development. So let's look at an alternative approach First let's reset our last merge and go to the previous state. spatel1-mn1:school-of-sre spatel1$ git reset --hard 60dc441 HEAD is now at 60dc441 adding master.txt file spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all * 60dc441 (HEAD - master) adding master.txt file | * 872a38f (b1) adding b1 file |/ * 7f3b00e adding file 2 * df2fb7a adding file 1 Option 2: Rebase. Now, instead of merging two branches which has a similar base (commit: 7f3b00e), let us rebase branch b1 on to current master. What this means is take branch b1 (from commit 7f3b00e to commit 872a38f) and rebase (put them on top of) master (60dc441). # Switch to b1 spatel1-mn1:school-of-sre spatel1$ git checkout b1 Switched to branch 'b1' # Rebase (b1 which is current branch) on master spatel1-mn1:school-of-sre spatel1$ git rebase master First, rewinding head to replay your work on top of it... Applying: adding b1 file # The result spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all * 5372c8f (HEAD - b1) adding b1 file * 60dc441 (master) adding master.txt file * 7f3b00e adding file 2 * df2fb7a adding file 1 You can see b1 which had 1 commit. That commit's parent was 7f3b00e . But since we rebase it on master ( 60dc441 ). That becomes the parent now. As a side effect, you also see it has become a single line of history. Now if we were to merge b1 into master , it would simply mean change master to point to 5372c8f which is b1 . Let's try it: # checkout to master since we want to merge code into master spatel1-mn1:school-of-sre spatel1$ git checkout master Switched to branch 'master' # the current history, where b1 is based on master spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all * 5372c8f (b1) adding b1 file * 60dc441 (HEAD - master) adding master.txt file * 7f3b00e adding file 2 * df2fb7a adding file 1 # Performing the merge, notice the fast-forward message spatel1-mn1:school-of-sre spatel1$ git merge b1 Updating 60dc441..5372c8f Fast-forward b1.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 b1.txt # The Result spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all * 5372c8f (HEAD - master, b1) adding b1 file * 60dc441 adding master.txt file * 7f3b00e adding file 2 * df2fb7a adding file 1 Now you see both b1 and master are pointing to the same commit. Your code has been merged to the master branch and it can be pushed. Also we have clean line of history! :D","title":"Working With Branches"},{"location":"git/branches/#working-with-branches","text":"Coming back to our local repo which has two commits. So far, what we have is a single line of history. Commits are chained in a single line. But sometimes you may have a need to work on two different features in parallel in the same repo. Now one option here could be making a new folder/repo with the same code and use that for another feature development. But there's a better way. Use branches. Since git follows tree like structure for commits, we can use branches to work on different sets of features. From a commit, two or more branches can be created and branches can also be merged. Using branches, there can exist multiple lines of histories and we can checkout to any of them and work on it. Checking out, as we discussed earlier, would simply mean replacing contents of the directory (repo) with contents snapshot at the checked out version. Let's create a branch and see how it looks like: spatel1-mn1:school-of-sre spatel1$ git branch b1 spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph * 7f3b00e (HEAD - master, b1) adding file 2 * df2fb7a adding file 1 We create a branch called b1 . Git log tells us that b1 also points to the last commit (7f3b00e) but the HEAD is still pointing to master. If you remember, HEAD points to the commit/reference wherever you are checkout to. So if we checkout to b1 , HEAD should point to that. Let's confirm: spatel1-mn1:school-of-sre spatel1$ git checkout b1 Switched to branch 'b1' spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph * 7f3b00e (HEAD - b1, master) adding file 2 * df2fb7a adding file 1 b1 still points to the same commit but HEAD now points to b1 . Since we create a branch at commit 7f3b00e , there will be two lines of histories starting this commit. Depending on which branch you are checked out on, the line of history will progress. At this moment, we are checked out on branch b1 , so making a new commit will advance branch reference b1 to that commit and current b1 commit will become its parent. Let's do that. # Creating a file and making a commit spatel1-mn1:school-of-sre spatel1$ echo I am a file in b1 branch b1.txt spatel1-mn1:school-of-sre spatel1$ git add b1.txt spatel1-mn1:school-of-sre spatel1$ git commit -m adding b1 file [b1 872a38f] adding b1 file 1 file changed, 1 insertion(+) create mode 100644 b1.txt # The new line of history spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph * 872a38f (HEAD - b1) adding b1 file * 7f3b00e (master) adding file 2 * df2fb7a adding file 1 spatel1-mn1:school-of-sre spatel1$ Do note that master is still pointing to the old commit it was pointing to. We can now checkout to master branch and make commits there. This will result in another line of history starting from commit 7f3b00e. # checkout to master branch spatel1-mn1:school-of-sre spatel1$ git checkout master Switched to branch 'master' # Creating a new commit on master branch spatel1-mn1:school-of-sre spatel1$ echo new file in master branch master.txt spatel1-mn1:school-of-sre spatel1$ git add master.txt spatel1-mn1:school-of-sre spatel1$ git commit -m adding master.txt file [master 60dc441] adding master.txt file 1 file changed, 1 insertion(+) create mode 100644 master.txt # The history line spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph * 60dc441 (HEAD - master) adding master.txt file * 7f3b00e adding file 2 * df2fb7a adding file 1 Notice how branch b1 is not visible here since we are checkout on master. Let's try to visualize both to get the whole picture: spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all * 60dc441 (HEAD - master) adding master.txt file | * 872a38f (b1) adding b1 file |/ * 7f3b00e adding file 2 * df2fb7a adding file 1 Above tree structure should make things clear. Notice a clear branch/fork on commit 7f3b00e. This is how we create branches. Now they both are two separate lines of history on which feature development can be done independently. To reiterate, internally, git is just a tree of commits. Branch names (human readable) are pointers to those commits in the tree. We use various git commands to work with the tree structure and references. Git accordingly modifies contents of our repo.","title":"Working With Branches"},{"location":"git/branches/#merges","text":"Now say the feature you were working on branch b1 is complete. And you need to merge it on master branch, where all the final version of code goes. So first you will checkout to branch master and then you will pull the latest code from upstream (eg: GitHub). Then you need to merge your code from b1 into master. And there could be two ways this can be done. Here is the current history: spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all * 60dc441 (HEAD - master) adding master.txt file | * 872a38f (b1) adding b1 file |/ * 7f3b00e adding file 2 * df2fb7a adding file 1 Option 1: Directly merge the branch. Merging the branch b1 into master will result in a new merge commit which will merge changes from two different lines of history and create a new commit of the result. spatel1-mn1:school-of-sre spatel1$ git merge b1 Merge made by the 'recursive' strategy. b1.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 b1.txt spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all * 8fc28f9 (HEAD - master) Merge branch 'b1' |\\ | * 872a38f (b1) adding b1 file * | 60dc441 adding master.txt file |/ * 7f3b00e adding file 2 * df2fb7a adding file 1 You can see a new merge commit created (8fc28f9). You will be prompted for the commit message. If there are a lot of branches in the repo, this result will end-up with a lot of merge commits. Which looks ugly compared to a single line of history of development. So let's look at an alternative approach First let's reset our last merge and go to the previous state. spatel1-mn1:school-of-sre spatel1$ git reset --hard 60dc441 HEAD is now at 60dc441 adding master.txt file spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all * 60dc441 (HEAD - master) adding master.txt file | * 872a38f (b1) adding b1 file |/ * 7f3b00e adding file 2 * df2fb7a adding file 1 Option 2: Rebase. Now, instead of merging two branches which has a similar base (commit: 7f3b00e), let us rebase branch b1 on to current master. What this means is take branch b1 (from commit 7f3b00e to commit 872a38f) and rebase (put them on top of) master (60dc441). # Switch to b1 spatel1-mn1:school-of-sre spatel1$ git checkout b1 Switched to branch 'b1' # Rebase (b1 which is current branch) on master spatel1-mn1:school-of-sre spatel1$ git rebase master First, rewinding head to replay your work on top of it... Applying: adding b1 file # The result spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all * 5372c8f (HEAD - b1) adding b1 file * 60dc441 (master) adding master.txt file * 7f3b00e adding file 2 * df2fb7a adding file 1 You can see b1 which had 1 commit. That commit's parent was 7f3b00e . But since we rebase it on master ( 60dc441 ). That becomes the parent now. As a side effect, you also see it has become a single line of history. Now if we were to merge b1 into master , it would simply mean change master to point to 5372c8f which is b1 . Let's try it: # checkout to master since we want to merge code into master spatel1-mn1:school-of-sre spatel1$ git checkout master Switched to branch 'master' # the current history, where b1 is based on master spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all * 5372c8f (b1) adding b1 file * 60dc441 (HEAD - master) adding master.txt file * 7f3b00e adding file 2 * df2fb7a adding file 1 # Performing the merge, notice the fast-forward message spatel1-mn1:school-of-sre spatel1$ git merge b1 Updating 60dc441..5372c8f Fast-forward b1.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 b1.txt # The Result spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph --all * 5372c8f (HEAD - master, b1) adding b1 file * 60dc441 adding master.txt file * 7f3b00e adding file 2 * df2fb7a adding file 1 Now you see both b1 and master are pointing to the same commit. Your code has been merged to the master branch and it can be pushed. Also we have clean line of history! :D","title":"Merges"},{"location":"git/git-basics/","text":"School Of SRE: Git Pre - Reads Have Git installed https://git-scm.com/downloads Have taken any git high level tutorial or following LinkedIn learning courses https://www.linkedin.com/learning/git-essential-training-the-basics/ https://www.linkedin.com/learning/git-branches-merges-and-remotes/ The Official Git Docs What to expect from this training As an engineer in the field of computer science, having knowledge of version control tools becomes almost a requirement. While there are a lot of version control tools that exist today, Git perhaps is the most used one and this course we will be working with Git. While this course does not start with Git 101 and expects basic knowledge of git as a prerequisite, it will reintroduce the git concepts known by you with details covering what is happening under the hood as you execute various git commands. So that next time you run a git command, you will be able to press enter more confidently! What is not covered under this training Advanced usage and specifics of internal implementation details of Git. Training Content Table of Contents Git Basics Working with Branches Git with Github Hooks Git Basics Though you might be aware already, let's revisit why we need a version control system. As the project grows and multiple developers start working on it, an efficient method for collaboration is warranted. Git helps the team collaborate easily and also maintains history of the changes happened with the codebase. Creating a Git Repo Any folder can be converted into a git repository. After executing the following command, we will see a .git folder within the folder, which makes our folder a git repository. All the magic that git does, .git folder is the enabler for the same. # creating an empty folder and changing current dir to it spatel1-mn1:~ spatel1$ cd /tmp spatel1-mn1:tmp spatel1$ mkdir school-of-sre spatel1-mn1:tmp spatel1$ cd school-of-sre/ # initialize a git repo spatel1-mn1:school-of-sre spatel1$ git init Initialized empty Git repository in /private/tmp/school-of-sre/.git/ As the output says, an empty git repo has been initialized in our folder. Let's take a look at what is there. spatel1-mn1:school-of-sre spatel1$ ls .git/ HEAD config description hooks info objects refs There are a bunch of folders and files in the .git folder. As I said, all these enables git to do its magic. We will look into some of these folders and files. But for now, what we have is an empty git repository. Tracking a File Now as you might already know, let us create a new file in our repo (we will refer to the folder as repo now.) And see git status spatel1-mn1:school-of-sre spatel1$ echo I am file 1 file1.txt spatel1-mn1:school-of-sre spatel1$ git status On branch master No commits yet Untracked files: (use git add file ... to include in what will be committed) file1.txt nothing added to commit but untracked files present (use git add to track) The current git status says No commits yet and there is one untracked file. Since we just created the file, git is not tracking that file. We explicitly need to ask git to track files and folders. (also checkout gitignore ) And how we do that is via git add command as suggested in the above output. Then we go ahead and create a commit. spatel1-mn1:school-of-sre spatel1$ git add file1.txt spatel1-mn1:school-of-sre spatel1$ git status On branch master No commits yet Changes to be committed: (use git rm --cached file ... to unstage) new file: file1.txt spatel1-mn1:school-of-sre spatel1$ git commit -m adding file 1 [master (root-commit) df2fb7a] adding file 1 1 file changed, 1 insertion(+) create mode 100644 file1.txt Notice how after adding the file, git status says Changes to be commited: . What it means is whatever is listed there, will be included in the next commit. Then we go ahead and create a commit, with an attached messaged via -m . More About a Commit Commit is a snapshot of the repo. Whenever a commit is made, a snapshot of the current state of repo (the folder) is taken and saved. Each commit has a unique ID. ( df2fb7a for the commit we made in the previous step). As we keep adding/changing more and more contents and keep making commits, all those snapshots are stored by git. Again, all this magic happens inside the .git folder. This is where all this snapshot or versions are stored. In an efficient manner. Adding More Changes Let us create one more file and commit the change. It would look the same as the previous commit we made. spatel1-mn1:school-of-sre spatel1$ echo I am file 2 file2.txt spatel1-mn1:school-of-sre spatel1$ git add file2.txt spatel1-mn1:school-of-sre spatel1$ git commit -m adding file 2 [master 7f3b00e] adding file 2 1 file changed, 1 insertion(+) create mode 100644 file2.txt A new commit with ID 7f3b00e has been created. You can issue git status at any time to see the state of the repository. **IMPORTANT: Note that commit IDs are long string (SHA) but we can refer to a commit by its initial few (8 or more) characters too. We will interchangeably using shorter and longer commit IDs.** Now that we have two commits, let's visualize them: spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph * 7f3b00e (HEAD - master) adding file 2 * df2fb7a adding file 1 git log , as the name suggests, prints the log of all the git commits. Here you see two additional arguments, --oneline prints the shorter version of the log, ie: the commit message only and not the person who made the commit and when. --graph prints it in graph format. Now at this moment the commits might look like just one in each line but all commits are stored as a tree like data structure internally by git. That means there can be two or more children commits of a given commit. And not just a single line of commits. We will look more into this part when we get to the Branches section. For now this is our commit history: df2fb7a === 7f3b00e Are commits really linked? As I just said, the two commits we just made are linked via tree like data structure and we saw how they are linked. But let's actually verify it. Everything in git is an object. Newly created files are stored as an object. Changes to file are stored as an objects and even commits are objects. To view contents of an object we can use the following command with the object's ID. We will take a look at content of the contents of the second commit spatel1-mn1:school-of-sre spatel1$ git cat-file -p 7f3b00e tree ebf3af44d253e5328340026e45a9fa9ae3ea1982 parent df2fb7a61f5d40c1191e0fdeb0fc5d6e7969685a author Sanket Patel spatel1@linkedin.com 1603273316 -0700 committer Sanket Patel spatel1@linkedin.com 1603273316 -0700 adding file 2 Take a note of parent attribute in the above output. It points to the commit id of the first commit we made. So this proves that they are linked! Additionally you can see the second commit's message in this object. As I said all this magic is enabled by .git folder and the object to which we are looking at also is in that folder. spatel1-mn1:school-of-sre spatel1$ ls .git/objects/7f/3b00eaa957815884198e2fdfec29361108d6a9 .git/objects/7f/3b00eaa957815884198e2fdfec29361108d6a9 It is stored in .git/objects/ folder. All the files and changes to them as well are stored in this folder. The Version Control part of Git We already can see two commits (versions) in our git log. One thing a version control tool gives you is ability to browse back and forth in history. For example: some of your users are running an old version of code and they are reporting an issue. In order to debug the issue, you need access to the old code. The one in your current repo is the latest code. In this example, you are working on the second commit (7f3b00e) and someone reported an issue with the code snapshot at commit (df2fb7a). This is how you would get access to the code at any older commit # Current contents, two files present patel1-mn1:school-of-sre spatel1$ ls file1.txt file2.txt # checking out to (an older) commit spatel1-mn1:school-of-sre spatel1$ git checkout df2fb7a Note: checking out 'df2fb7a'. You are in 'detached HEAD' state. You can look around, make experimental changes and commit them, and you can discard any commits you make in this state without impacting any branches by performing another checkout. If you want to create a new branch to retain commits you create, you may do so (now or later) by using -b with the checkout command again. Example: git checkout -b new-branch-name HEAD is now at df2fb7a adding file 1 # checking contents, can verify it has old contents spatel1-mn1:school-of-sre spatel1$ ls file1.txt So this is how we would get access to old versions/snapshots. All we need is a reference to that snapshot. Upon executing git checkout ... , what git does for you is use the .git folder, see what was the state of things (files and folders) at that version/reference and replace the contents of current directory with those contents. The then-existing content will no longer be present in the local dir (repo) but we can and will still get access to them because they are tracked via git commit and .git folder has them stored/tracked. Reference I mention in the previous section that we need a reference to the version. By default, git repo is made of tree of commits. And each commit has a unique IDs. But the unique ID is not the only thing we can reference commits via. There are multiple ways to reference commits. For example: HEAD is a reference to current commit. Whatever commit your repo is checked out at, HEAD will point to that. HEAD~1 is reference to previous commit. So while checking out previous version in section above, we could have done git checkout HEAD~1 . Similarly, master is also a reference (to a branch). Since git uses tree like structure to store commits, there of course will be branches. And the default branch is called master . Master (or any branch reference) will point to the latest commit in the branch. Even though we have checked out to the previous commit in out repo, master still points to the latest commit. And we can get back to the latest version by checkout at master reference spatel1-mn1:school-of-sre spatel1$ git checkout master Previous HEAD position was df2fb7a adding file 1 Switched to branch 'master' # now we will see latest code, with two files spatel1-mn1:school-of-sre spatel1$ ls file1.txt file2.txt Note, instead of master in above command, we could have used commit's ID as well. References and The Magic Let's look at the state of things. Two commits, master and HEAD references are pointing to the latest commit spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph * 7f3b00e (HEAD - master) adding file 2 * df2fb7a adding file 1 The magic? Let's examine these files: spatel1-mn1:school-of-sre spatel1$ cat .git/refs/heads/master 7f3b00eaa957815884198e2fdfec29361108d6a9 Viola! Where master is pointing to is stored in a file. Whenever git needs to know where master reference is pointing to, or if git needs to update where master points, it just needs to update the file above. So when you create a new commit, a new commit is created on top of the current commit and the master file is updated with the new commit's ID. Similary, for HEAD reference: spatel1-mn1:school-of-sre spatel1$ cat .git/HEAD ref: refs/heads/master We can see HEAD is pointing to a reference called refs/heads/master . So HEAD will point where ever the master points. Little Adventure We discussed how git will update the files as we execute commands. But let's try to do it ourselves, by hand, and see what happens. spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph * 7f3b00e (HEAD - master) adding file 2 * df2fb7a adding file 1 Now let's change master to point to the previous/first commit. spatel1-mn1:school-of-sre spatel1$ echo df2fb7a61f5d40c1191e0fdeb0fc5d6e7969685a .git/refs/heads/master spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph * df2fb7a (HEAD - master) adding file 1 # RESETTING TO ORIGINAL spatel1-mn1:school-of-sre spatel1$ echo 7f3b00eaa957815884198e2fdfec29361108d6a9 .git/refs/heads/master spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph * 7f3b00e (HEAD - master) adding file 2 * df2fb7a adding file 1 We just edited the master reference file and now we can see only the first commit in git log. Undoing the change to the file brings the state back to original. Not so much of magic, isn't it?","title":"Git Basics"},{"location":"git/git-basics/#school-of-sre-git","text":"","title":"School Of SRE: Git"},{"location":"git/git-basics/#pre-reads","text":"Have Git installed https://git-scm.com/downloads Have taken any git high level tutorial or following LinkedIn learning courses https://www.linkedin.com/learning/git-essential-training-the-basics/ https://www.linkedin.com/learning/git-branches-merges-and-remotes/ The Official Git Docs","title":"Pre - Reads"},{"location":"git/git-basics/#what-to-expect-from-this-training","text":"As an engineer in the field of computer science, having knowledge of version control tools becomes almost a requirement. While there are a lot of version control tools that exist today, Git perhaps is the most used one and this course we will be working with Git. While this course does not start with Git 101 and expects basic knowledge of git as a prerequisite, it will reintroduce the git concepts known by you with details covering what is happening under the hood as you execute various git commands. So that next time you run a git command, you will be able to press enter more confidently!","title":"What to expect from this training"},{"location":"git/git-basics/#what-is-not-covered-under-this-training","text":"Advanced usage and specifics of internal implementation details of Git.","title":"What is not covered under this training"},{"location":"git/git-basics/#training-content","text":"","title":"Training Content"},{"location":"git/git-basics/#table-of-contents","text":"Git Basics Working with Branches Git with Github Hooks","title":"Table of Contents"},{"location":"git/git-basics/#git-basics","text":"Though you might be aware already, let's revisit why we need a version control system. As the project grows and multiple developers start working on it, an efficient method for collaboration is warranted. Git helps the team collaborate easily and also maintains history of the changes happened with the codebase.","title":"Git Basics"},{"location":"git/git-basics/#creating-a-git-repo","text":"Any folder can be converted into a git repository. After executing the following command, we will see a .git folder within the folder, which makes our folder a git repository. All the magic that git does, .git folder is the enabler for the same. # creating an empty folder and changing current dir to it spatel1-mn1:~ spatel1$ cd /tmp spatel1-mn1:tmp spatel1$ mkdir school-of-sre spatel1-mn1:tmp spatel1$ cd school-of-sre/ # initialize a git repo spatel1-mn1:school-of-sre spatel1$ git init Initialized empty Git repository in /private/tmp/school-of-sre/.git/ As the output says, an empty git repo has been initialized in our folder. Let's take a look at what is there. spatel1-mn1:school-of-sre spatel1$ ls .git/ HEAD config description hooks info objects refs There are a bunch of folders and files in the .git folder. As I said, all these enables git to do its magic. We will look into some of these folders and files. But for now, what we have is an empty git repository.","title":"Creating a Git Repo"},{"location":"git/git-basics/#tracking-a-file","text":"Now as you might already know, let us create a new file in our repo (we will refer to the folder as repo now.) And see git status spatel1-mn1:school-of-sre spatel1$ echo I am file 1 file1.txt spatel1-mn1:school-of-sre spatel1$ git status On branch master No commits yet Untracked files: (use git add file ... to include in what will be committed) file1.txt nothing added to commit but untracked files present (use git add to track) The current git status says No commits yet and there is one untracked file. Since we just created the file, git is not tracking that file. We explicitly need to ask git to track files and folders. (also checkout gitignore ) And how we do that is via git add command as suggested in the above output. Then we go ahead and create a commit. spatel1-mn1:school-of-sre spatel1$ git add file1.txt spatel1-mn1:school-of-sre spatel1$ git status On branch master No commits yet Changes to be committed: (use git rm --cached file ... to unstage) new file: file1.txt spatel1-mn1:school-of-sre spatel1$ git commit -m adding file 1 [master (root-commit) df2fb7a] adding file 1 1 file changed, 1 insertion(+) create mode 100644 file1.txt Notice how after adding the file, git status says Changes to be commited: . What it means is whatever is listed there, will be included in the next commit. Then we go ahead and create a commit, with an attached messaged via -m .","title":"Tracking a File"},{"location":"git/git-basics/#more-about-a-commit","text":"Commit is a snapshot of the repo. Whenever a commit is made, a snapshot of the current state of repo (the folder) is taken and saved. Each commit has a unique ID. ( df2fb7a for the commit we made in the previous step). As we keep adding/changing more and more contents and keep making commits, all those snapshots are stored by git. Again, all this magic happens inside the .git folder. This is where all this snapshot or versions are stored. In an efficient manner.","title":"More About a Commit"},{"location":"git/git-basics/#adding-more-changes","text":"Let us create one more file and commit the change. It would look the same as the previous commit we made. spatel1-mn1:school-of-sre spatel1$ echo I am file 2 file2.txt spatel1-mn1:school-of-sre spatel1$ git add file2.txt spatel1-mn1:school-of-sre spatel1$ git commit -m adding file 2 [master 7f3b00e] adding file 2 1 file changed, 1 insertion(+) create mode 100644 file2.txt A new commit with ID 7f3b00e has been created. You can issue git status at any time to see the state of the repository. **IMPORTANT: Note that commit IDs are long string (SHA) but we can refer to a commit by its initial few (8 or more) characters too. We will interchangeably using shorter and longer commit IDs.** Now that we have two commits, let's visualize them: spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph * 7f3b00e (HEAD - master) adding file 2 * df2fb7a adding file 1 git log , as the name suggests, prints the log of all the git commits. Here you see two additional arguments, --oneline prints the shorter version of the log, ie: the commit message only and not the person who made the commit and when. --graph prints it in graph format. Now at this moment the commits might look like just one in each line but all commits are stored as a tree like data structure internally by git. That means there can be two or more children commits of a given commit. And not just a single line of commits. We will look more into this part when we get to the Branches section. For now this is our commit history: df2fb7a === 7f3b00e","title":"Adding More Changes"},{"location":"git/git-basics/#are-commits-really-linked","text":"As I just said, the two commits we just made are linked via tree like data structure and we saw how they are linked. But let's actually verify it. Everything in git is an object. Newly created files are stored as an object. Changes to file are stored as an objects and even commits are objects. To view contents of an object we can use the following command with the object's ID. We will take a look at content of the contents of the second commit spatel1-mn1:school-of-sre spatel1$ git cat-file -p 7f3b00e tree ebf3af44d253e5328340026e45a9fa9ae3ea1982 parent df2fb7a61f5d40c1191e0fdeb0fc5d6e7969685a author Sanket Patel spatel1@linkedin.com 1603273316 -0700 committer Sanket Patel spatel1@linkedin.com 1603273316 -0700 adding file 2 Take a note of parent attribute in the above output. It points to the commit id of the first commit we made. So this proves that they are linked! Additionally you can see the second commit's message in this object. As I said all this magic is enabled by .git folder and the object to which we are looking at also is in that folder. spatel1-mn1:school-of-sre spatel1$ ls .git/objects/7f/3b00eaa957815884198e2fdfec29361108d6a9 .git/objects/7f/3b00eaa957815884198e2fdfec29361108d6a9 It is stored in .git/objects/ folder. All the files and changes to them as well are stored in this folder.","title":"Are commits really linked?"},{"location":"git/git-basics/#the-version-control-part-of-git","text":"We already can see two commits (versions) in our git log. One thing a version control tool gives you is ability to browse back and forth in history. For example: some of your users are running an old version of code and they are reporting an issue. In order to debug the issue, you need access to the old code. The one in your current repo is the latest code. In this example, you are working on the second commit (7f3b00e) and someone reported an issue with the code snapshot at commit (df2fb7a). This is how you would get access to the code at any older commit # Current contents, two files present patel1-mn1:school-of-sre spatel1$ ls file1.txt file2.txt # checking out to (an older) commit spatel1-mn1:school-of-sre spatel1$ git checkout df2fb7a Note: checking out 'df2fb7a'. You are in 'detached HEAD' state. You can look around, make experimental changes and commit them, and you can discard any commits you make in this state without impacting any branches by performing another checkout. If you want to create a new branch to retain commits you create, you may do so (now or later) by using -b with the checkout command again. Example: git checkout -b new-branch-name HEAD is now at df2fb7a adding file 1 # checking contents, can verify it has old contents spatel1-mn1:school-of-sre spatel1$ ls file1.txt So this is how we would get access to old versions/snapshots. All we need is a reference to that snapshot. Upon executing git checkout ... , what git does for you is use the .git folder, see what was the state of things (files and folders) at that version/reference and replace the contents of current directory with those contents. The then-existing content will no longer be present in the local dir (repo) but we can and will still get access to them because they are tracked via git commit and .git folder has them stored/tracked.","title":"The Version Control part of Git"},{"location":"git/git-basics/#reference","text":"I mention in the previous section that we need a reference to the version. By default, git repo is made of tree of commits. And each commit has a unique IDs. But the unique ID is not the only thing we can reference commits via. There are multiple ways to reference commits. For example: HEAD is a reference to current commit. Whatever commit your repo is checked out at, HEAD will point to that. HEAD~1 is reference to previous commit. So while checking out previous version in section above, we could have done git checkout HEAD~1 . Similarly, master is also a reference (to a branch). Since git uses tree like structure to store commits, there of course will be branches. And the default branch is called master . Master (or any branch reference) will point to the latest commit in the branch. Even though we have checked out to the previous commit in out repo, master still points to the latest commit. And we can get back to the latest version by checkout at master reference spatel1-mn1:school-of-sre spatel1$ git checkout master Previous HEAD position was df2fb7a adding file 1 Switched to branch 'master' # now we will see latest code, with two files spatel1-mn1:school-of-sre spatel1$ ls file1.txt file2.txt Note, instead of master in above command, we could have used commit's ID as well.","title":"Reference"},{"location":"git/git-basics/#references-and-the-magic","text":"Let's look at the state of things. Two commits, master and HEAD references are pointing to the latest commit spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph * 7f3b00e (HEAD - master) adding file 2 * df2fb7a adding file 1 The magic? Let's examine these files: spatel1-mn1:school-of-sre spatel1$ cat .git/refs/heads/master 7f3b00eaa957815884198e2fdfec29361108d6a9 Viola! Where master is pointing to is stored in a file. Whenever git needs to know where master reference is pointing to, or if git needs to update where master points, it just needs to update the file above. So when you create a new commit, a new commit is created on top of the current commit and the master file is updated with the new commit's ID. Similary, for HEAD reference: spatel1-mn1:school-of-sre spatel1$ cat .git/HEAD ref: refs/heads/master We can see HEAD is pointing to a reference called refs/heads/master . So HEAD will point where ever the master points.","title":"References and The Magic"},{"location":"git/git-basics/#little-adventure","text":"We discussed how git will update the files as we execute commands. But let's try to do it ourselves, by hand, and see what happens. spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph * 7f3b00e (HEAD - master) adding file 2 * df2fb7a adding file 1 Now let's change master to point to the previous/first commit. spatel1-mn1:school-of-sre spatel1$ echo df2fb7a61f5d40c1191e0fdeb0fc5d6e7969685a .git/refs/heads/master spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph * df2fb7a (HEAD - master) adding file 1 # RESETTING TO ORIGINAL spatel1-mn1:school-of-sre spatel1$ echo 7f3b00eaa957815884198e2fdfec29361108d6a9 .git/refs/heads/master spatel1-mn1:school-of-sre spatel1$ git log --oneline --graph * 7f3b00e (HEAD - master) adding file 2 * df2fb7a adding file 1 We just edited the master reference file and now we can see only the first commit in git log. Undoing the change to the file brings the state back to original. Not so much of magic, isn't it?","title":"Little Adventure"},{"location":"git/github-hooks/","text":"Git with Github Till now all the operations we did were in our local repo while git also helps us in a collaborative environment. GitHub is one place on the internet where you can centrally host your git repos and collaborate with other developers. Most of the workflow will remain the same as we discussed, with addition of couple of things: Pull: to pull latest changes from github (the central) repo Push: to push your changes to github repo so that it's available to all people GitHub has written nice guides and tutorials about this and you can refer them here: GitHub Hello World Git Handbook Hooks Git has another nice feature called hooks. Hooks are basically scripts which will be called when a certain event happens. Here is where hooks are located: spatel1-mn1:school-of-sre spatel1$ ls .git/hooks/ applypatch-msg.sample fsmonitor-watchman.sample pre-applypatch.sample pre-push.sample pre-receive.sample update.sample commit-msg.sample post-update.sample pre-commit.sample pre-rebase.sample prepare-commit-msg.sample Names are self explanatory. These hooks are useful when you want to do certain things when a certain event happens. Ie: if you want to run tests before pushing code, you would want to setup pre-push hooks. Let's try to create a pre commit hook. spatel1-mn1:school-of-sre spatel1$ echo echo this is from pre commit hook .git/hooks/pre-commit spatel1-mn1:school-of-sre spatel1$ chmod +x .git/hooks/pre-commit We basically create a file called pre-commit in hooks folder and make it executable. Now if we make a commit, we should see the message getting printed. spatel1-mn1:school-of-sre spatel1$ echo sample file sample.txt spatel1-mn1:school-of-sre spatel1$ git add sample.txt spatel1-mn1:school-of-sre spatel1$ git commit -m adding sample file this is from pre commit hook # ===== THE MESSAGE FROM HOOK EXECUTION [master 9894e05] adding sample file 1 file changed, 1 insertion(+) create mode 100644 sample.txt What next from here? There are a lot of git commands and features which we have not explored here. But with the base built-up, be sure to explore concepts like Cherrypick Squash Amend Stash Reset","title":"Github and Hooks"},{"location":"git/github-hooks/#git-with-github","text":"Till now all the operations we did were in our local repo while git also helps us in a collaborative environment. GitHub is one place on the internet where you can centrally host your git repos and collaborate with other developers. Most of the workflow will remain the same as we discussed, with addition of couple of things: Pull: to pull latest changes from github (the central) repo Push: to push your changes to github repo so that it's available to all people GitHub has written nice guides and tutorials about this and you can refer them here: GitHub Hello World Git Handbook","title":"Git with Github"},{"location":"git/github-hooks/#hooks","text":"Git has another nice feature called hooks. Hooks are basically scripts which will be called when a certain event happens. Here is where hooks are located: spatel1-mn1:school-of-sre spatel1$ ls .git/hooks/ applypatch-msg.sample fsmonitor-watchman.sample pre-applypatch.sample pre-push.sample pre-receive.sample update.sample commit-msg.sample post-update.sample pre-commit.sample pre-rebase.sample prepare-commit-msg.sample Names are self explanatory. These hooks are useful when you want to do certain things when a certain event happens. Ie: if you want to run tests before pushing code, you would want to setup pre-push hooks. Let's try to create a pre commit hook. spatel1-mn1:school-of-sre spatel1$ echo echo this is from pre commit hook .git/hooks/pre-commit spatel1-mn1:school-of-sre spatel1$ chmod +x .git/hooks/pre-commit We basically create a file called pre-commit in hooks folder and make it executable. Now if we make a commit, we should see the message getting printed. spatel1-mn1:school-of-sre spatel1$ echo sample file sample.txt spatel1-mn1:school-of-sre spatel1$ git add sample.txt spatel1-mn1:school-of-sre spatel1$ git commit -m adding sample file this is from pre commit hook # ===== THE MESSAGE FROM HOOK EXECUTION [master 9894e05] adding sample file 1 file changed, 1 insertion(+) create mode 100644 sample.txt","title":"Hooks"},{"location":"git/github-hooks/#what-next-from-here","text":"There are a lot of git commands and features which we have not explored here. But with the base built-up, be sure to explore concepts like Cherrypick Squash Amend Stash Reset","title":"What next from here?"},{"location":"python_web/intro/","text":"School of SRE: Python and The Web Pre - Reads Basic understanding of python language. Basic familiarity with flask framework. What to expect from this training This course is divided into two high level parts. In the first part, assuming familiarity with python language\u2019s basic operations and syntax usage, we will dive a little deeper into understanding python as a language. We will compare python with other programming languages that you might already know like Java and C. We will also explore concepts of Python objects and with help of that, explore python features like decorators. In the second part which will revolve around the web, and also assume familiarity with the Flask framework, we will start from the socket module and work with HTTP requests. This will demystify how frameworks like flask work internally. And to introduce SRE flavour to the course, we will design, develop and deploy (in theory) a URL shortening application. We will emphasize parts of the whole process that are more important as an SRE of the said app/service. What is not covered under this training Extensive knowledge of python internals and advanced python. Training Content Lab Environment Setup Have latest version of python installed TOC The Python Language Some Python Concepts Python Gotchas Python and Web Sockets Flask The URL Shortening App Design Scaling The App Monitoring The App The Python Language Assuming you know a little bit of C/C++ and Java, let's try to discuss the following questions in context of those two languages and python. You might have heard that C/C++ is a compiled language while python is an interpreted language. Generally, with compiled language we first compile the program and then run the executable while in case of python we run the source code directly like python hello_world.py . While Java, being an interpreted language, still has a separate compilation step and then its run. So what's really the difference? Compiled vs. Interpreted This might sound a little weird to you: python, in a way is a compiled language! Python has a compiler built-in! It is obvious in the case of java since we compile it using a separate command ie: javac helloWorld.java and it will produce a .class file which we know as a bytecode . Well, python is very similar to that. One difference here is that there is no separate compile command/binary needed to run a python program. What is the difference then, between java and python? Well, Java's compiler is more strict and sophisticated. As you might know Java is a statically typed language. So the compiler is written in a way that it can verify types related errors during compile time. While python being a dynamic language, types are not known until a program is run. So in a way, python compiler is dumb (or, less strict). But there indeed is a compile step involved when a python program is run. You might have seen python bytecode files with .pyc extension. Here is how you can see bytecode for a given python program. # Create a Hello World spatel1-mn1:tmp spatel1$ echo print('hello world') hello_world.py # Making sure it runs spatel1-mn1:tmp spatel1$ python3 hello_world.py hello world # The bytecode of the given program spatel1-mn1:tmp spatel1$ python -m dis hello_world.py 1 0 LOAD_NAME 0 (print) 2 LOAD_CONST 0 ('hello world') 4 CALL_FUNCTION 1 6 POP_TOP 8 LOAD_CONST 1 (None) 10 RETURN_VALUE Read more about dis module here Now coming to C/C++, there of course is a compiler. But the output is different than what java/python compiler would produce. Compiling a C program would produce what we also know as machine code . As opposed to bytecode. Running The Programs We know compilation is involved in all 3 languages we are discussing. Just that the compilers are different in nature and they output different types of content. In case of C/C++, the output is machine code which can be directly read by your operating system. When you execute that program, your OS will know how exactly to run it. But this is not the case with bytecode. Those bytecodes are language specific. Python has its own set of bytecode defined (more in dis module) and so does java. So naturally, your operating system will not know how to run it. To run this bytecode, we have something called Virtual Machines. Ie: The JVM or the Python VM (CPython, Jython). These so called Virtual Machines are the programs which can read the bytecode and run it on a given operating system. Python has multiple VMs available. Cpython is a python VM implemented in C language, similarly Jython is a Java implementation of python VM. At the end of the day, what they should be capable of is to understand python language syntax, be able to compile it to bytecode and be able to run that bytecode. You can implement a python VM in any language! (And people do so, just because it can be done) The Operating System +------------------------------------+ | | | | | | hello_world.py Python bytecode | Python VM Process | | | +----------------+ +----------------+ | +----------------+ | |print(... | COMPILE |LOAD_CONST... | | |Reads bytecode | | | +--------------- + +------------------- +line by line | | | | | | | |and executes. | | | | | | | | | | +----------------+ +----------------+ | +----------------+ | | | | | | | hello_world.c OS Specific machinecode | A New Process | | | +----------------+ +----------------+ | +----------------+ | |void main() { | COMPILE | binary contents| | | binary contents| | | +--------------- + +------------------- + | | | | | | | | | | | | | | | | | | +----------------+ +----------------+ | +----------------+ | | (binary contents | | runs as is) | | | | | +------------------------------------+ Two things to note for above diagram: Generally, when we run a python program, a python VM process is started which reads the python source code, compiles it to byte code and run it in a single step. Compiling is not a separate step. Shown only for illustration purpose. Binaries generated for C like languages are not exactly run as is. Since there are multiple types of binaries (eg: ELF), there are more complicated steps involved in order to run a binary but we will not go into that since all that is done at OS level.","title":"Intro"},{"location":"python_web/intro/#school-of-sre-python-and-the-web","text":"","title":"School of SRE: Python and The Web"},{"location":"python_web/intro/#pre-reads","text":"Basic understanding of python language. Basic familiarity with flask framework.","title":"Pre - Reads"},{"location":"python_web/intro/#what-to-expect-from-this-training","text":"This course is divided into two high level parts. In the first part, assuming familiarity with python language\u2019s basic operations and syntax usage, we will dive a little deeper into understanding python as a language. We will compare python with other programming languages that you might already know like Java and C. We will also explore concepts of Python objects and with help of that, explore python features like decorators. In the second part which will revolve around the web, and also assume familiarity with the Flask framework, we will start from the socket module and work with HTTP requests. This will demystify how frameworks like flask work internally. And to introduce SRE flavour to the course, we will design, develop and deploy (in theory) a URL shortening application. We will emphasize parts of the whole process that are more important as an SRE of the said app/service.","title":"What to expect from this training"},{"location":"python_web/intro/#what-is-not-covered-under-this-training","text":"Extensive knowledge of python internals and advanced python.","title":"What is not covered under this training"},{"location":"python_web/intro/#training-content","text":"","title":"Training Content"},{"location":"python_web/intro/#lab-environment-setup","text":"Have latest version of python installed","title":"Lab Environment Setup"},{"location":"python_web/intro/#toc","text":"The Python Language Some Python Concepts Python Gotchas Python and Web Sockets Flask The URL Shortening App Design Scaling The App Monitoring The App","title":"TOC"},{"location":"python_web/intro/#the-python-language","text":"Assuming you know a little bit of C/C++ and Java, let's try to discuss the following questions in context of those two languages and python. You might have heard that C/C++ is a compiled language while python is an interpreted language. Generally, with compiled language we first compile the program and then run the executable while in case of python we run the source code directly like python hello_world.py . While Java, being an interpreted language, still has a separate compilation step and then its run. So what's really the difference?","title":"The Python Language"},{"location":"python_web/intro/#compiled-vs-interpreted","text":"This might sound a little weird to you: python, in a way is a compiled language! Python has a compiler built-in! It is obvious in the case of java since we compile it using a separate command ie: javac helloWorld.java and it will produce a .class file which we know as a bytecode . Well, python is very similar to that. One difference here is that there is no separate compile command/binary needed to run a python program. What is the difference then, between java and python? Well, Java's compiler is more strict and sophisticated. As you might know Java is a statically typed language. So the compiler is written in a way that it can verify types related errors during compile time. While python being a dynamic language, types are not known until a program is run. So in a way, python compiler is dumb (or, less strict). But there indeed is a compile step involved when a python program is run. You might have seen python bytecode files with .pyc extension. Here is how you can see bytecode for a given python program. # Create a Hello World spatel1-mn1:tmp spatel1$ echo print('hello world') hello_world.py # Making sure it runs spatel1-mn1:tmp spatel1$ python3 hello_world.py hello world # The bytecode of the given program spatel1-mn1:tmp spatel1$ python -m dis hello_world.py 1 0 LOAD_NAME 0 (print) 2 LOAD_CONST 0 ('hello world') 4 CALL_FUNCTION 1 6 POP_TOP 8 LOAD_CONST 1 (None) 10 RETURN_VALUE Read more about dis module here Now coming to C/C++, there of course is a compiler. But the output is different than what java/python compiler would produce. Compiling a C program would produce what we also know as machine code . As opposed to bytecode.","title":"Compiled vs. Interpreted"},{"location":"python_web/intro/#running-the-programs","text":"We know compilation is involved in all 3 languages we are discussing. Just that the compilers are different in nature and they output different types of content. In case of C/C++, the output is machine code which can be directly read by your operating system. When you execute that program, your OS will know how exactly to run it. But this is not the case with bytecode. Those bytecodes are language specific. Python has its own set of bytecode defined (more in dis module) and so does java. So naturally, your operating system will not know how to run it. To run this bytecode, we have something called Virtual Machines. Ie: The JVM or the Python VM (CPython, Jython). These so called Virtual Machines are the programs which can read the bytecode and run it on a given operating system. Python has multiple VMs available. Cpython is a python VM implemented in C language, similarly Jython is a Java implementation of python VM. At the end of the day, what they should be capable of is to understand python language syntax, be able to compile it to bytecode and be able to run that bytecode. You can implement a python VM in any language! (And people do so, just because it can be done) The Operating System +------------------------------------+ | | | | | | hello_world.py Python bytecode | Python VM Process | | | +----------------+ +----------------+ | +----------------+ | |print(... | COMPILE |LOAD_CONST... | | |Reads bytecode | | | +--------------- + +------------------- +line by line | | | | | | | |and executes. | | | | | | | | | | +----------------+ +----------------+ | +----------------+ | | | | | | | hello_world.c OS Specific machinecode | A New Process | | | +----------------+ +----------------+ | +----------------+ | |void main() { | COMPILE | binary contents| | | binary contents| | | +--------------- + +------------------- + | | | | | | | | | | | | | | | | | | +----------------+ +----------------+ | +----------------+ | | (binary contents | | runs as is) | | | | | +------------------------------------+ Two things to note for above diagram: Generally, when we run a python program, a python VM process is started which reads the python source code, compiles it to byte code and run it in a single step. Compiling is not a separate step. Shown only for illustration purpose. Binaries generated for C like languages are not exactly run as is. Since there are multiple types of binaries (eg: ELF), there are more complicated steps involved in order to run a binary but we will not go into that since all that is done at OS level.","title":"Running The Programs"},{"location":"python_web/python-concepts/","text":"Some Python Concepts Though you are expected to know python and its syntax at basic level, let us discuss some fundamental concepts that will help you understand the python language better. Everything in Python is an object. That includes the functions, lists, dicts, classes, modules, a running function (instance of function definition), everything. In the CPython, it would mean there is an underlying struct variable for each object. In python's current execution context, all the variables are stored in a dict. It'd be a string to object mapping. If you have a function and a float variable defined in the current context, here is how it is handled internally. float_number=42.0 def foo_func(): ... pass ... # NOTICE HOW VARIABLE NAMES ARE STRINGS, stored in a dict locals() {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': class '_frozen_importlib.BuiltinImporter' , '__spec__': None, '__annotations__': {}, '__builtins__': module 'builtins' (built-in) , 'float_number': 42.0, 'foo_func': function foo_func at 0x1055847a0 } Python Functions Since functions too are objects, we can see what all attributes a function contains as following def hello(name): ... print(f Hello, {name}! ) ... dir(hello) ['__annotations__', '__call__', '__class__', '__closure__', '__code__', '__defaults__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__get__', '__getattribute__', '__globals__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__kwdefaults__', '__le__', '__lt__', '__module__', '__name__', '__ne__', '__new__', '__qualname__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__'] While there are a lot of them, let's look at some interesting ones globals This attribute, as the name suggests, has references of global variables. If you ever need to know what all global variables are in the scope of this function, this will tell you. See how the function start seeing the new variable in globals hello.__globals__ {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': class '_frozen_importlib.BuiltinImporter' , '__spec__': None, '__annotations__': {}, '__builtins__': module 'builtins' (built-in) , 'hello': function hello at 0x7fe4e82554c0 } # adding new global variable GLOBAL= g_val hello.__globals__ {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': class '_frozen_importlib.BuiltinImporter' , '__spec__': None, '__annotations__': {}, '__builtins__': module 'builtins' (built-in) , 'hello': function hello at 0x7fe4e82554c0 , 'GLOBAL': 'g_val'} code This is an interesting one! As everything in python is an object, this includes the bytecode too. The compiled python bytecode is a python code object. Which is accessible via __code__ attribute here. A function has an associated code object which carries some interesting information. # the file in which function is defined # stdin here since this is run in an interpreter hello.__code__.co_filename ' stdin ' # number of arguments the function takes hello.__code__.co_argcount 1 # local variable names hello.__code__.co_varnames ('name',) # the function code's compiled bytecode hello.__code__.co_code b't\\x00d\\x01|\\x00\\x9b\\x00d\\x02\\x9d\\x03\\x83\\x01\\x01\\x00d\\x00S\\x00' There are more code attributes which you can enlist by dir(hello.__code__) Decorators Related to functions, python has another feature called decorators. Let's see how that works, keeping everything is an object in mind. Here is a sample decorator: def deco(func): ... def inner(): ... print( before ) ... func() ... print( after ) ... return inner ... @deco ... def hello_world(): ... print( hello world ) ... hello_world() before hello world after Here @deco syntax is used to decorate the hello_world function. It is essentially same as doing def hello_world(): ... print( hello world ) ... hello_world = deco(hello_world) What goes inside the deco function might seem complex. Let's try to uncover it. Function hello_world is created It is passed to deco function deco create a new function This new function is calls hello_world function And does a couple other things deco returns the newly created function hello_world is replaced with above function Let's visualize it for better understanding BEFORE function_object (ID: 100) hello_world +--------------------+ + |print( hello_world )| | | | +-------------- | | | | +--------------------+ WHAT DECORATOR DOES creates a new function (ID: 101) +---------------------------------+ |input arg: function with id: 100 | | | |print( before ) | |call function object with id 100 | |print( after ) | | | +---------------------------^-----+ | | AFTER | | | hello_world +-------------+ Note how the hello_world name points to a new function object but that new function object knows the reference (ID) of the original function. Some Gotchas While it is very quick to build prototypes in python and there are tons of libraries available, as the codebase complexity increases, type errors become more common and will get hard to deal with. (There are solutions to that problem like type annotations in python. Checkout mypy .) Because python is dynamically typed language, that means all types are determined at runtime. And that makes python run very slow compared to other statically typed languages. Python has something called GIL (global interpreter lock) which is a limiting factor for utilizing multiple CPI cores for parallel computation. Some weird things that python does: https://github.com/satwikkansal/wtfpython","title":"Some Python Concepts"},{"location":"python_web/python-concepts/#some-python-concepts","text":"Though you are expected to know python and its syntax at basic level, let us discuss some fundamental concepts that will help you understand the python language better. Everything in Python is an object. That includes the functions, lists, dicts, classes, modules, a running function (instance of function definition), everything. In the CPython, it would mean there is an underlying struct variable for each object. In python's current execution context, all the variables are stored in a dict. It'd be a string to object mapping. If you have a function and a float variable defined in the current context, here is how it is handled internally. float_number=42.0 def foo_func(): ... pass ... # NOTICE HOW VARIABLE NAMES ARE STRINGS, stored in a dict locals() {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': class '_frozen_importlib.BuiltinImporter' , '__spec__': None, '__annotations__': {}, '__builtins__': module 'builtins' (built-in) , 'float_number': 42.0, 'foo_func': function foo_func at 0x1055847a0 }","title":"Some Python Concepts"},{"location":"python_web/python-concepts/#python-functions","text":"Since functions too are objects, we can see what all attributes a function contains as following def hello(name): ... print(f Hello, {name}! ) ... dir(hello) ['__annotations__', '__call__', '__class__', '__closure__', '__code__', '__defaults__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__get__', '__getattribute__', '__globals__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__kwdefaults__', '__le__', '__lt__', '__module__', '__name__', '__ne__', '__new__', '__qualname__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__'] While there are a lot of them, let's look at some interesting ones","title":"Python Functions"},{"location":"python_web/python-concepts/#globals","text":"This attribute, as the name suggests, has references of global variables. If you ever need to know what all global variables are in the scope of this function, this will tell you. See how the function start seeing the new variable in globals hello.__globals__ {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': class '_frozen_importlib.BuiltinImporter' , '__spec__': None, '__annotations__': {}, '__builtins__': module 'builtins' (built-in) , 'hello': function hello at 0x7fe4e82554c0 } # adding new global variable GLOBAL= g_val hello.__globals__ {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': class '_frozen_importlib.BuiltinImporter' , '__spec__': None, '__annotations__': {}, '__builtins__': module 'builtins' (built-in) , 'hello': function hello at 0x7fe4e82554c0 , 'GLOBAL': 'g_val'}","title":"globals"},{"location":"python_web/python-concepts/#code","text":"This is an interesting one! As everything in python is an object, this includes the bytecode too. The compiled python bytecode is a python code object. Which is accessible via __code__ attribute here. A function has an associated code object which carries some interesting information. # the file in which function is defined # stdin here since this is run in an interpreter hello.__code__.co_filename ' stdin ' # number of arguments the function takes hello.__code__.co_argcount 1 # local variable names hello.__code__.co_varnames ('name',) # the function code's compiled bytecode hello.__code__.co_code b't\\x00d\\x01|\\x00\\x9b\\x00d\\x02\\x9d\\x03\\x83\\x01\\x01\\x00d\\x00S\\x00' There are more code attributes which you can enlist by dir(hello.__code__)","title":"code"},{"location":"python_web/python-concepts/#decorators","text":"Related to functions, python has another feature called decorators. Let's see how that works, keeping everything is an object in mind. Here is a sample decorator: def deco(func): ... def inner(): ... print( before ) ... func() ... print( after ) ... return inner ... @deco ... def hello_world(): ... print( hello world ) ... hello_world() before hello world after Here @deco syntax is used to decorate the hello_world function. It is essentially same as doing def hello_world(): ... print( hello world ) ... hello_world = deco(hello_world) What goes inside the deco function might seem complex. Let's try to uncover it. Function hello_world is created It is passed to deco function deco create a new function This new function is calls hello_world function And does a couple other things deco returns the newly created function hello_world is replaced with above function Let's visualize it for better understanding BEFORE function_object (ID: 100) hello_world +--------------------+ + |print( hello_world )| | | | +-------------- | | | | +--------------------+ WHAT DECORATOR DOES creates a new function (ID: 101) +---------------------------------+ |input arg: function with id: 100 | | | |print( before ) | |call function object with id 100 | |print( after ) | | | +---------------------------^-----+ | | AFTER | | | hello_world +-------------+ Note how the hello_world name points to a new function object but that new function object knows the reference (ID) of the original function.","title":"Decorators"},{"location":"python_web/python-concepts/#some-gotchas","text":"While it is very quick to build prototypes in python and there are tons of libraries available, as the codebase complexity increases, type errors become more common and will get hard to deal with. (There are solutions to that problem like type annotations in python. Checkout mypy .) Because python is dynamically typed language, that means all types are determined at runtime. And that makes python run very slow compared to other statically typed languages. Python has something called GIL (global interpreter lock) which is a limiting factor for utilizing multiple CPI cores for parallel computation. Some weird things that python does: https://github.com/satwikkansal/wtfpython","title":"Some Gotchas"},{"location":"python_web/python-web-flask/","text":"Python, Web amd Flask Back in the old days, websites were simple. They were simple static html contents. A webserver would be listening on a defined port and according to the HTTP request received, it would read files from disk and return them in response. But since then, complexity has evolved and websites are now dynamic. Depending on the request, multiple operations need to be performed like reading from database or calling other API and finally returning some response (HTML data, JSON content etc.) Since serving web requests is no longer a simple task like reading files from disk and return contents, we need to process each http request, perform some operations programmatically and construct a response. Sockets Though we have frameworks like flask, HTTP is still a protocol that works over TCP protocol. So let us setup a TCP server and send an HTTP request and inspect the request's payload. Note that this is not a tutorial on socket programming but what we are doing here is inspecting HTTP protocol at its ground level and look at what its contents look like. (Ref: Socket Programming in Python (Guide) on RealPython ) import socket HOST = '127.0.0.1' # Standard loopback interface address (localhost) PORT = 65432 # Port to listen on (non-privileged ports are 1023) with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind((HOST, PORT)) s.listen() conn, addr = s.accept() with conn: print('Connected by', addr) while True: data = conn.recv(1024) if not data: break print(data) Then we open localhost:65432 in our web browser and following would be the output: Connected by ('127.0.0.1', 54719) b'GET / HTTP/1.1\\r\\nHost: localhost:65432\\r\\nConnection: keep-alive\\r\\nDNT: 1\\r\\nUpgrade-Insecure-Requests: 1\\r\\nUser-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36 Edg/85.0.564.44\\r\\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9\\r\\nSec-Fetch-Site: none\\r\\nSec-Fetch-Mode: navigate\\r\\nSec-Fetch-User: ?1\\r\\nSec-Fetch-Dest: document\\r\\nAccept-Encoding: gzip, deflate, br\\r\\nAccept-Language: en-US,en;q=0.9\\r\\n\\r\\n' Examine closely and the content will look like the HTTP protocol's format. ie: HTTP_METHOD URI_PATH HTTP_VERSION HEADERS_SEPARATED_BY_SEPARATOR So though it's a blob of bytes, knowing http protocol specification , you can parse that string (ie: split by \\r\\n ) and get meaningful information out of it. Flask Flask, and other such frameworks does pretty much what we just discussed in the last section (with added more sophistication). They listen on a port on a TCP socket, receive an HTTP request, parse the data according to protocol format and make it available to you in a convenient manner. ie: you can access headers in flask by request.headers which is made available to you by splitting above payload by /r/n , as defined in http protocol. Another example: we register routes in flask by @app.route(\"/hello\") . What flask will do is maintain a registry internally which will map /hello with the function you decorated with. Now whenever a request comes with the /hello route (second component in the first line, split by space), flask calls the registered function and returns whatever the function returned. Same with all other web frameworks in other languages too. They all work on similar principles. What they basically do is understand the HTTP protocol, parses the HTTP request data and gives us programmers a nice interface to work with HTTP requests. Not so much of magic, innit?","title":"Python, Web and Flask"},{"location":"python_web/python-web-flask/#python-web-amd-flask","text":"Back in the old days, websites were simple. They were simple static html contents. A webserver would be listening on a defined port and according to the HTTP request received, it would read files from disk and return them in response. But since then, complexity has evolved and websites are now dynamic. Depending on the request, multiple operations need to be performed like reading from database or calling other API and finally returning some response (HTML data, JSON content etc.) Since serving web requests is no longer a simple task like reading files from disk and return contents, we need to process each http request, perform some operations programmatically and construct a response.","title":"Python, Web amd Flask"},{"location":"python_web/python-web-flask/#sockets","text":"Though we have frameworks like flask, HTTP is still a protocol that works over TCP protocol. So let us setup a TCP server and send an HTTP request and inspect the request's payload. Note that this is not a tutorial on socket programming but what we are doing here is inspecting HTTP protocol at its ground level and look at what its contents look like. (Ref: Socket Programming in Python (Guide) on RealPython ) import socket HOST = '127.0.0.1' # Standard loopback interface address (localhost) PORT = 65432 # Port to listen on (non-privileged ports are 1023) with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind((HOST, PORT)) s.listen() conn, addr = s.accept() with conn: print('Connected by', addr) while True: data = conn.recv(1024) if not data: break print(data) Then we open localhost:65432 in our web browser and following would be the output: Connected by ('127.0.0.1', 54719) b'GET / HTTP/1.1\\r\\nHost: localhost:65432\\r\\nConnection: keep-alive\\r\\nDNT: 1\\r\\nUpgrade-Insecure-Requests: 1\\r\\nUser-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36 Edg/85.0.564.44\\r\\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9\\r\\nSec-Fetch-Site: none\\r\\nSec-Fetch-Mode: navigate\\r\\nSec-Fetch-User: ?1\\r\\nSec-Fetch-Dest: document\\r\\nAccept-Encoding: gzip, deflate, br\\r\\nAccept-Language: en-US,en;q=0.9\\r\\n\\r\\n' Examine closely and the content will look like the HTTP protocol's format. ie: HTTP_METHOD URI_PATH HTTP_VERSION HEADERS_SEPARATED_BY_SEPARATOR So though it's a blob of bytes, knowing http protocol specification , you can parse that string (ie: split by \\r\\n ) and get meaningful information out of it.","title":"Sockets"},{"location":"python_web/python-web-flask/#flask","text":"Flask, and other such frameworks does pretty much what we just discussed in the last section (with added more sophistication). They listen on a port on a TCP socket, receive an HTTP request, parse the data according to protocol format and make it available to you in a convenient manner. ie: you can access headers in flask by request.headers which is made available to you by splitting above payload by /r/n , as defined in http protocol. Another example: we register routes in flask by @app.route(\"/hello\") . What flask will do is maintain a registry internally which will map /hello with the function you decorated with. Now whenever a request comes with the /hello route (second component in the first line, split by space), flask calls the registered function and returns whatever the function returned. Same with all other web frameworks in other languages too. They all work on similar principles. What they basically do is understand the HTTP protocol, parses the HTTP request data and gives us programmers a nice interface to work with HTTP requests. Not so much of magic, innit?","title":"Flask"},{"location":"python_web/sre-conclusion/","text":"SRE Parts of The App and Conclusion Scaling The App The design and development is just a part of the journey. We will need to setup continuous integration and continuous delivery pipelines sooner or later. And we have to deploy this app somewhere. Initially we can start with deploying this app on one virtual machine on any cloud provider. But this is a Single point of failure which is something we never allow as an SRE (or even as an engineer). So an improvement here can be having multiple instances of applications deployed behind a load balancer. This certainly prevents problems of one machine going down. Scaling here would mean adding more instances behind the load balancer. But this is scalable upto only a certain point. After that, other bottlenecks in the system will start appearing. ie: DB will become the bottleneck, or perhaps the load balancer itself. How do you know what is the bottleneck? You need to have observability into each aspects of the application architecture. Only after you have metrics, you will be able to know what is going wrong where. What gets measured, gets fixed! Get deeper insights into scaling from School Of SRE's Scalability module and post going through it, apply your learnings and takeaways to this app. Think how will we make this app geographically distributed and highly available and scalable. Monitoring Strategy Once we have our application deployed. It will be working ok. But not forever. Reliability is in the title of our job and we make systems reliable by making the design in a certain way. But things still will go down. Machines will fail. Disks will behave weirdly. Buggy code will get pushed to production. And all these possible scenarios will make the system less reliable. So what do we do? We monitor! We keep an eye on the system's health and if anything is not going as expected, we want ourselves to get alerted. Now let's think in terms of the given url shortening app. We need to monitor it. And we would want to get notified in case something goes wrong. But we first need to decide what is that something that we want to keep an eye on. Since it's a web app serving HTTP requests, we want to keep an eye on HTTP Status codes and latencies Request volume again is a good candidate, if the app is receiving an unusual amount of traffic, something might be off. We also want to keep an eye on the database so depending on the database solution chosen. Query times, volumes, disk usage etc. Finally, there also needs to be some external monitoring which runs periodic tests from devices outside of your data centers. This emulates customers and ensures that from customer point of view, the system is working as expected. SRE Use-cases In the world of SRE, python is a widely used language. For small scripts and tooling developed for various purposes. Since tooling developed by SRE works with critical pieces of infrastructure and has great power (to bring things down), it is important to know what you are doing while using a programming language and its features. Also it is equally important to know the language and its characteristics while debugging the issues. As an SRE having a deeper understanding of python language, it has helped me a lot to debug very sneaky bugs and be generally more aware and informed while making certain design decisions. While developing tools may or may not be part of SRE job, supporting tools or services is more likely to be a daily duty. Building an application or tool is just a small part of productionization. While there is certainly that goes in the design of the application itself to make it more robust, as an SRE you are responsible for its reliability and stability once it is deployed and running. And to ensure that, you\u2019d need to understand the application first and then come up with a strategy to monitor it properly and be prepared for various failure scenarios. Optional Exercises Make a decorator that will cache function return values depending on input parameters. Host the URL shortening app on any cloud provider. Setup monitoring using many of the tools available like catchpoint, datadog etc. Create a minimal flask-like framework on top of TCP sockets. Conclusion This module, in the first part, aims to make you more aware of the things that will happen when you choose python as your programming language and what happens when you run a python program. With the knowledge of how python handles things internally as objects, lot of seemingly magic things in python will start to make more sense. The second part will first explain how a framework like flask works using the existing knowledge of protocols like TCP and HTTP. It then touches the whole lifecycle of an application development lifecycle including the SRE parts of it. While the design and areas in architecture considered will not be exhaustive, it will give a good overview of things that are also important being an SRE and why they are important.","title":"SRE Aspects of The App and Conclusion"},{"location":"python_web/sre-conclusion/#sre-parts-of-the-app-and-conclusion","text":"","title":"SRE Parts of The App and Conclusion"},{"location":"python_web/sre-conclusion/#scaling-the-app","text":"The design and development is just a part of the journey. We will need to setup continuous integration and continuous delivery pipelines sooner or later. And we have to deploy this app somewhere. Initially we can start with deploying this app on one virtual machine on any cloud provider. But this is a Single point of failure which is something we never allow as an SRE (or even as an engineer). So an improvement here can be having multiple instances of applications deployed behind a load balancer. This certainly prevents problems of one machine going down. Scaling here would mean adding more instances behind the load balancer. But this is scalable upto only a certain point. After that, other bottlenecks in the system will start appearing. ie: DB will become the bottleneck, or perhaps the load balancer itself. How do you know what is the bottleneck? You need to have observability into each aspects of the application architecture. Only after you have metrics, you will be able to know what is going wrong where. What gets measured, gets fixed! Get deeper insights into scaling from School Of SRE's Scalability module and post going through it, apply your learnings and takeaways to this app. Think how will we make this app geographically distributed and highly available and scalable.","title":"Scaling The App"},{"location":"python_web/sre-conclusion/#monitoring-strategy","text":"Once we have our application deployed. It will be working ok. But not forever. Reliability is in the title of our job and we make systems reliable by making the design in a certain way. But things still will go down. Machines will fail. Disks will behave weirdly. Buggy code will get pushed to production. And all these possible scenarios will make the system less reliable. So what do we do? We monitor! We keep an eye on the system's health and if anything is not going as expected, we want ourselves to get alerted. Now let's think in terms of the given url shortening app. We need to monitor it. And we would want to get notified in case something goes wrong. But we first need to decide what is that something that we want to keep an eye on. Since it's a web app serving HTTP requests, we want to keep an eye on HTTP Status codes and latencies Request volume again is a good candidate, if the app is receiving an unusual amount of traffic, something might be off. We also want to keep an eye on the database so depending on the database solution chosen. Query times, volumes, disk usage etc. Finally, there also needs to be some external monitoring which runs periodic tests from devices outside of your data centers. This emulates customers and ensures that from customer point of view, the system is working as expected.","title":"Monitoring Strategy"},{"location":"python_web/sre-conclusion/#sre-use-cases","text":"In the world of SRE, python is a widely used language. For small scripts and tooling developed for various purposes. Since tooling developed by SRE works with critical pieces of infrastructure and has great power (to bring things down), it is important to know what you are doing while using a programming language and its features. Also it is equally important to know the language and its characteristics while debugging the issues. As an SRE having a deeper understanding of python language, it has helped me a lot to debug very sneaky bugs and be generally more aware and informed while making certain design decisions. While developing tools may or may not be part of SRE job, supporting tools or services is more likely to be a daily duty. Building an application or tool is just a small part of productionization. While there is certainly that goes in the design of the application itself to make it more robust, as an SRE you are responsible for its reliability and stability once it is deployed and running. And to ensure that, you\u2019d need to understand the application first and then come up with a strategy to monitor it properly and be prepared for various failure scenarios.","title":"SRE Use-cases"},{"location":"python_web/sre-conclusion/#optional-exercises","text":"Make a decorator that will cache function return values depending on input parameters. Host the URL shortening app on any cloud provider. Setup monitoring using many of the tools available like catchpoint, datadog etc. Create a minimal flask-like framework on top of TCP sockets.","title":"Optional Exercises"},{"location":"python_web/sre-conclusion/#conclusion","text":"This module, in the first part, aims to make you more aware of the things that will happen when you choose python as your programming language and what happens when you run a python program. With the knowledge of how python handles things internally as objects, lot of seemingly magic things in python will start to make more sense. The second part will first explain how a framework like flask works using the existing knowledge of protocols like TCP and HTTP. It then touches the whole lifecycle of an application development lifecycle including the SRE parts of it. While the design and areas in architecture considered will not be exhaustive, it will give a good overview of things that are also important being an SRE and why they are important.","title":"Conclusion"},{"location":"python_web/url-shorten-app/","text":"The URL Shortening App Let's build a very simple URL shortening app using flask and try to incorporate all aspects of the development process including the reliability aspects. We will not be building the UI and we will come up with a minimal set of API that will be enough for the app to function well. Design We don't jump directly to coding. First thing we do is gather requirements. Come up with an approach. Have the approach/design reviewed by peers. Evolve, iterate, document the decisions and tradeoffs. And then finally implement. While we will not do the full blown design document here, we will raise certain questions here that are important to the design. 1. High Level Operations and API Endpoints Since it's a URL shortening app, we will need an API for generating the shorten link given an original link. And an API/Endpoint which will accept the shorten link and redirect to original URL. We are not including the user aspect of the app to keep things minimal. These two API should make app functional and usable by anyone. 2. How to shorten? Given a url, we will need to generate a shortened version of it. One approach could be using random characters for each link. Another thing that can be done is to use some sort of hashing algorithm. The benefit here is we will reuse the same hash for the same link. ie: if lot of people are shortening https://www.linkedin.com they all will have the same value, compared to multiple entries in DB if chosen random characters. What about hash collisions? Even in random characters approach, though there is a less probability, hash collisions can happen. And we need to be mindful of them. In that case we might want to prepend/append the string with some random value to avoid conflict. Also, choice of hash algorithm matters. We will need to analyze algorithms. Their CPU requirements and their characteristics. Choose one that suits the most. 3. Is URL Valid? Given a URL to shorten, how do we verify if the URL is valid? Do we even verify or validate? One basic check that can be done is see if the URL matches a regex of a URL. To go even further we can try opening/visiting the URL. But there are certain gotchas here. We need to define success criteria. ie: HTTP 200 means it is valid. What is the URL is in private network? What if URL is temporarily down? 4. Storage Finally, storage. Where will we store the data that we will generate over time? There are multiple database solutions available and we will need to choose the one that suits this app the most. Relational database like MySQL would be a fair choice but be sure to checkout School of SRE's database section for deeper insights into making a more informed decision. 5. Other We are not accounting for users into our app and other possible features like rate limiting, customized links etc but it will eventually come up with time. Depending on the requirements, they too might need to get incorporated. The minimal working code is given below for reference but I'd encourage you to come up with your own. from flask import Flask, redirect, request from hashlib import md5 app = Flask( url_shortener ) mapping = {} @app.route( /shorten , methods=[ POST ]) def shorten(): global mapping payload = request.json if url not in payload: return Missing URL Parameter , 400 # TODO: check if URL is valid hash_ = md5() hash_.update(payload[ url ].encode()) digest = hash_.hexdigest()[:5] # limiting to 5 chars. Less the limit more the chances of collission if digest not in mapping: mapping[digest] = payload[ url ] return f Shortened: r/{digest}\\n else: # TODO: check for hash collission return f Already exists: r/{digest}\\n @app.route( /r/ hash_ ) def redirect_(hash_): if hash_ not in mapping: return URL Not Found , 404 return redirect(mapping[hash_]) if __name__ == __main__ : app.run(debug=True) OUTPUT: === SHORTENING spatel1-mn1:tmp spatel1$ curl localhost:5000/shorten -H content-type: application/json --data '{ url : https://linkedin.com }' Shortened: r/a62a4 === REDIRECTING, notice the response code 302 and the location header spatel1-mn1:tmp spatel1$ curl localhost:5000/r/a62a4 -v * Uses proxy env variable NO_PROXY == '127.0.0.1' * Trying ::1... * TCP_NODELAY set * Connection failed * connect to ::1 port 5000 failed: Connection refused * Trying 127.0.0.1... * TCP_NODELAY set * Connected to localhost (127.0.0.1) port 5000 (#0) GET /r/a62a4 HTTP/1.1 Host: localhost:5000 User-Agent: curl/7.64.1 Accept: */* * HTTP 1.0, assume close after body HTTP/1.0 302 FOUND Content-Type: text/html; charset=utf-8 Content-Length: 247 Location: https://linkedin.com Server: Werkzeug/0.15.4 Python/3.7.7 Date: Tue, 27 Oct 2020 09:37:12 GMT !DOCTYPE HTML PUBLIC -//W3C//DTD HTML 3.2 Final//EN title Redirecting... /title h1 Redirecting... /h1 * Closing connection 0 p You should be redirected automatically to target URL: a href= https://linkedin.com https://linkedin.com /a . If not click the link.","title":"The URL Shortening App"},{"location":"python_web/url-shorten-app/#the-url-shortening-app","text":"Let's build a very simple URL shortening app using flask and try to incorporate all aspects of the development process including the reliability aspects. We will not be building the UI and we will come up with a minimal set of API that will be enough for the app to function well.","title":"The URL Shortening App"},{"location":"python_web/url-shorten-app/#design","text":"We don't jump directly to coding. First thing we do is gather requirements. Come up with an approach. Have the approach/design reviewed by peers. Evolve, iterate, document the decisions and tradeoffs. And then finally implement. While we will not do the full blown design document here, we will raise certain questions here that are important to the design.","title":"Design"},{"location":"python_web/url-shorten-app/#1-high-level-operations-and-api-endpoints","text":"Since it's a URL shortening app, we will need an API for generating the shorten link given an original link. And an API/Endpoint which will accept the shorten link and redirect to original URL. We are not including the user aspect of the app to keep things minimal. These two API should make app functional and usable by anyone.","title":"1. High Level Operations and API Endpoints"},{"location":"python_web/url-shorten-app/#2-how-to-shorten","text":"Given a url, we will need to generate a shortened version of it. One approach could be using random characters for each link. Another thing that can be done is to use some sort of hashing algorithm. The benefit here is we will reuse the same hash for the same link. ie: if lot of people are shortening https://www.linkedin.com they all will have the same value, compared to multiple entries in DB if chosen random characters. What about hash collisions? Even in random characters approach, though there is a less probability, hash collisions can happen. And we need to be mindful of them. In that case we might want to prepend/append the string with some random value to avoid conflict. Also, choice of hash algorithm matters. We will need to analyze algorithms. Their CPU requirements and their characteristics. Choose one that suits the most.","title":"2. How to shorten?"},{"location":"python_web/url-shorten-app/#3-is-url-valid","text":"Given a URL to shorten, how do we verify if the URL is valid? Do we even verify or validate? One basic check that can be done is see if the URL matches a regex of a URL. To go even further we can try opening/visiting the URL. But there are certain gotchas here. We need to define success criteria. ie: HTTP 200 means it is valid. What is the URL is in private network? What if URL is temporarily down?","title":"3. Is URL Valid?"},{"location":"python_web/url-shorten-app/#4-storage","text":"Finally, storage. Where will we store the data that we will generate over time? There are multiple database solutions available and we will need to choose the one that suits this app the most. Relational database like MySQL would be a fair choice but be sure to checkout School of SRE's database section for deeper insights into making a more informed decision.","title":"4. Storage"},{"location":"python_web/url-shorten-app/#5-other","text":"We are not accounting for users into our app and other possible features like rate limiting, customized links etc but it will eventually come up with time. Depending on the requirements, they too might need to get incorporated. The minimal working code is given below for reference but I'd encourage you to come up with your own. from flask import Flask, redirect, request from hashlib import md5 app = Flask( url_shortener ) mapping = {} @app.route( /shorten , methods=[ POST ]) def shorten(): global mapping payload = request.json if url not in payload: return Missing URL Parameter , 400 # TODO: check if URL is valid hash_ = md5() hash_.update(payload[ url ].encode()) digest = hash_.hexdigest()[:5] # limiting to 5 chars. Less the limit more the chances of collission if digest not in mapping: mapping[digest] = payload[ url ] return f Shortened: r/{digest}\\n else: # TODO: check for hash collission return f Already exists: r/{digest}\\n @app.route( /r/ hash_ ) def redirect_(hash_): if hash_ not in mapping: return URL Not Found , 404 return redirect(mapping[hash_]) if __name__ == __main__ : app.run(debug=True) OUTPUT: === SHORTENING spatel1-mn1:tmp spatel1$ curl localhost:5000/shorten -H content-type: application/json --data '{ url : https://linkedin.com }' Shortened: r/a62a4 === REDIRECTING, notice the response code 302 and the location header spatel1-mn1:tmp spatel1$ curl localhost:5000/r/a62a4 -v * Uses proxy env variable NO_PROXY == '127.0.0.1' * Trying ::1... * TCP_NODELAY set * Connection failed * connect to ::1 port 5000 failed: Connection refused * Trying 127.0.0.1... * TCP_NODELAY set * Connected to localhost (127.0.0.1) port 5000 (#0) GET /r/a62a4 HTTP/1.1 Host: localhost:5000 User-Agent: curl/7.64.1 Accept: */* * HTTP 1.0, assume close after body HTTP/1.0 302 FOUND Content-Type: text/html; charset=utf-8 Content-Length: 247 Location: https://linkedin.com Server: Werkzeug/0.15.4 Python/3.7.7 Date: Tue, 27 Oct 2020 09:37:12 GMT !DOCTYPE HTML PUBLIC -//W3C//DTD HTML 3.2 Final//EN title Redirecting... /title h1 Redirecting... /h1 * Closing connection 0 p You should be redirected automatically to target URL: a href= https://linkedin.com https://linkedin.com /a . If not click the link.","title":"5. Other"},{"location":"systems_design/availability/","text":"HA - Availability - Common \u201cNines\u201d Availability is generally expressed as \u201cNines\u201d, common \u2018Nines\u2019 are listed below. Availability % Downtime per year Downtime per month Downtime per week Downtime per day 99%(Two Nines) 3.65 days 7.31 hours 1.68 hours 14.40 minutes 99.5%(Two and a half Nines) 1.83 days 3.65 hours 50.40 minutes 7.20 minutes 99.9%(Three Nines) 8.77 hours 43.83 minutes 10.08 minutes 1.44 minutes 99.95%(Three and a half Nines) 4.38 hours 21.92 minutes 5.04 minutes 43.20 seconds 99.99%(Four Nines) 52.60 minutes 4.38 minutes 1.01 minutes 8.64 seconds 99.995%(Four and a half Nines) 26.30 minutes 2.19 minutes 30.24 seconds 4.32 seconds 99.999%(Five Nines) 5.26 minutes 26.30 seconds 6.05 seconds 864.0 ms Refer https://en.wikipedia.org/wiki/High_availability#Percentage_calculation HA - Availability Serial Components A System with components is operating in the series If failure of a part leads to the combination becoming inoperable. For example if LB in our architecture fails, all access to app tiers will fail. LB and app tiers are connected serially. The combined availability of the system is the product of individual components availability A = Ax x Ay x \u2026.. Refer http://www.eventhelix.com/RealtimeMantra/FaultHandling/system_reliability_availability.htm HA - Availability Parallel Components A System with components is operating in parallel If failure of a part leads to the other part taking over the operations of the failed part. If we have more than one LB and if rest of the LBs can take over the traffic during one LB failure then LBs are operating in parallel The combined availability of the system is A = 1 - ( (1-Ax) x (1-Ax) x \u2026.. ) Refer http://www.eventhelix.com/RealtimeMantra/FaultHandling/system_reliability_availability.htm HA - Core Principles Elimination of single points of failure (SPOF) This means adding redundancy to the system so that the failure of a component does not mean failure of the entire system. Reliable crossover In redundant systems, the crossover point itself tends to become a single point of failure. Reliable systems must provide for reliable crossover. Detection of failures as they occur If the two principles above are observed, then a user may never see a failure Refer https://en.wikipedia.org/wiki/High_availability#Principles HA - SPOF WHAT: Never implement and always eliminate single points of failure. WHEN TO USE: During architecture reviews and new designs. HOW TO USE: Identify single instances on architectural diagrams. Strive for active/active configurations. At the very least we should have a standby to take control when active instances fail. WHY: Maximize availability through multiple instances. KEY TAKEAWAYS: Strive for active/active rather than active/passive solutions. Use load balancers to balance traffic across instances of a service. Use control services with active/passive instances for patterns that require singletons. HA - Reliable Crossover WHAT: Ensure when system components failover they do so reliably. WHEN TO USE: During architecture reviews, failure modeling, and designs. HOW TO USE: Identify how available a system is during the crossover and ensure it is within acceptable limits. WHY: Maximize availability and ensure data handling semantics are preserved. KEY TAKEAWAYS: Strive for active/active rather than active/passive solutions, they have a lesser risk of cross over being unreliable. Use LB and right load balancing methods to ensure reliable failover. Model and build your data systems to ensure data is correctly handled when crossover happens. Generally DB systems follow active/passive semantics for writes. Masters accept writes and when master goes down, follower is promoted to master(active from being passive) to accept writes. We have to be careful here that the cutover never introduces more than one masters. This problem is called a split brain. SRE Use cases SRE works on deciding an acceptable SLA and make sure system is available to achieve the SLA SRE is involved in architecture design right from building the data center to make sure site is not affected by network switch, hardware, power or software failures SRE also run mock drills of failures to see how the system behaves in uncharted territory and comes up with a plan to improve availability if there are misses. https://engineering.linkedin.com/blog/2017/11/resilience-engineering-at-linkedin-with-project-waterbear Post our understanding about HA, our architecture diagram looks something like this below","title":"Availability"},{"location":"systems_design/availability/#ha-availability-common-nines","text":"Availability is generally expressed as \u201cNines\u201d, common \u2018Nines\u2019 are listed below. Availability % Downtime per year Downtime per month Downtime per week Downtime per day 99%(Two Nines) 3.65 days 7.31 hours 1.68 hours 14.40 minutes 99.5%(Two and a half Nines) 1.83 days 3.65 hours 50.40 minutes 7.20 minutes 99.9%(Three Nines) 8.77 hours 43.83 minutes 10.08 minutes 1.44 minutes 99.95%(Three and a half Nines) 4.38 hours 21.92 minutes 5.04 minutes 43.20 seconds 99.99%(Four Nines) 52.60 minutes 4.38 minutes 1.01 minutes 8.64 seconds 99.995%(Four and a half Nines) 26.30 minutes 2.19 minutes 30.24 seconds 4.32 seconds 99.999%(Five Nines) 5.26 minutes 26.30 seconds 6.05 seconds 864.0 ms","title":"HA - Availability - Common \u201cNines\u201d"},{"location":"systems_design/availability/#refer","text":"https://en.wikipedia.org/wiki/High_availability#Percentage_calculation","title":"Refer"},{"location":"systems_design/availability/#ha-availability-serial-components","text":"A System with components is operating in the series If failure of a part leads to the combination becoming inoperable. For example if LB in our architecture fails, all access to app tiers will fail. LB and app tiers are connected serially. The combined availability of the system is the product of individual components availability A = Ax x Ay x \u2026..","title":"HA - Availability Serial Components"},{"location":"systems_design/availability/#refer_1","text":"http://www.eventhelix.com/RealtimeMantra/FaultHandling/system_reliability_availability.htm","title":"Refer"},{"location":"systems_design/availability/#ha-availability-parallel-components","text":"A System with components is operating in parallel If failure of a part leads to the other part taking over the operations of the failed part. If we have more than one LB and if rest of the LBs can take over the traffic during one LB failure then LBs are operating in parallel The combined availability of the system is A = 1 - ( (1-Ax) x (1-Ax) x \u2026.. )","title":"HA - Availability Parallel Components"},{"location":"systems_design/availability/#refer_2","text":"http://www.eventhelix.com/RealtimeMantra/FaultHandling/system_reliability_availability.htm","title":"Refer"},{"location":"systems_design/availability/#ha-core-principles","text":"Elimination of single points of failure (SPOF) This means adding redundancy to the system so that the failure of a component does not mean failure of the entire system. Reliable crossover In redundant systems, the crossover point itself tends to become a single point of failure. Reliable systems must provide for reliable crossover. Detection of failures as they occur If the two principles above are observed, then a user may never see a failure","title":"HA - Core Principles"},{"location":"systems_design/availability/#refer_3","text":"https://en.wikipedia.org/wiki/High_availability#Principles","title":"Refer"},{"location":"systems_design/availability/#ha-spof","text":"WHAT: Never implement and always eliminate single points of failure. WHEN TO USE: During architecture reviews and new designs. HOW TO USE: Identify single instances on architectural diagrams. Strive for active/active configurations. At the very least we should have a standby to take control when active instances fail. WHY: Maximize availability through multiple instances. KEY TAKEAWAYS: Strive for active/active rather than active/passive solutions. Use load balancers to balance traffic across instances of a service. Use control services with active/passive instances for patterns that require singletons.","title":"HA - SPOF"},{"location":"systems_design/availability/#ha-reliable-crossover","text":"WHAT: Ensure when system components failover they do so reliably. WHEN TO USE: During architecture reviews, failure modeling, and designs. HOW TO USE: Identify how available a system is during the crossover and ensure it is within acceptable limits. WHY: Maximize availability and ensure data handling semantics are preserved. KEY TAKEAWAYS: Strive for active/active rather than active/passive solutions, they have a lesser risk of cross over being unreliable. Use LB and right load balancing methods to ensure reliable failover. Model and build your data systems to ensure data is correctly handled when crossover happens. Generally DB systems follow active/passive semantics for writes. Masters accept writes and when master goes down, follower is promoted to master(active from being passive) to accept writes. We have to be careful here that the cutover never introduces more than one masters. This problem is called a split brain.","title":"HA - Reliable Crossover"},{"location":"systems_design/availability/#sre-use-cases","text":"SRE works on deciding an acceptable SLA and make sure system is available to achieve the SLA SRE is involved in architecture design right from building the data center to make sure site is not affected by network switch, hardware, power or software failures SRE also run mock drills of failures to see how the system behaves in uncharted territory and comes up with a plan to improve availability if there are misses. https://engineering.linkedin.com/blog/2017/11/resilience-engineering-at-linkedin-with-project-waterbear Post our understanding about HA, our architecture diagram looks something like this below","title":"SRE Use cases"},{"location":"systems_design/conclusion/","text":"Conclusion Armed with these principles, we hope the course will give a fresh perspective to design software systems. It might be over engineering to get all this on day zero. But some are really important from day 0 like eliminating single points of failure, making scalable services by just increasing replicas. As a bottleneck is reached, we can split code by services, shard data to scale. As the organisation matures, bringing in chaos engineering to measure how systems react to failure will help in designing robust software systems.","title":"Conclusion"},{"location":"systems_design/conclusion/#conclusion","text":"Armed with these principles, we hope the course will give a fresh perspective to design software systems. It might be over engineering to get all this on day zero. But some are really important from day 0 like eliminating single points of failure, making scalable services by just increasing replicas. As a bottleneck is reached, we can split code by services, shard data to scale. As the organisation matures, bringing in chaos engineering to measure how systems react to failure will help in designing robust software systems.","title":"Conclusion"},{"location":"systems_design/fault-tolerance/","text":"Fault Tolerance Failures are not avoidable in any system and will happen all the time, hence we need to build systems that can tolerate failures or recover from them. In systems, failure is the norm rather than the exception. \"Anything that can go wrong will go wrong\u201d -- Murphy\u2019s Law \u201cComplex systems contain changing mixtures of failures latent within them\u201d -- How Complex Systems Fail. Fault Tolerance - Failure Metrics Common failure metrics that get measured and tracked for any system. Mean time to repair (MTTR): The average time to repair and restore a failed system. Mean time between failures (MTBF): The average operational time between one device failure or system breakdown and the next. Mean time to failure (MTTF): The average time a device or system is expected to function before it fails. Mean time to detect (MTTD): The average time between the onset of a problem and when the organization detects it. Mean time to investigate (MTTI): The average time between the detection of an incident and when the organization begins to investigate its cause and solution. Mean time to restore service (MTRS): The average elapsed time from the detection of an incident until the affected system or component is again available to users. Mean time between system incidents (MTBSI): The average elapsed time between the detection of two consecutive incidents. MTBSI can be calculated by adding MTBF and MTRS (MTBSI = MTBF + MTRS). Failure rate: Another reliability metric, which measures the frequency with which a component or system fails. It is expressed as a number of failures over a unit of time. Refer https://www.splunk.com/en_us/data-insider/what-is-mean-time-to-repair.html Fault Tolerance - Fault Isolation Terms Systems should have a short circuit. Say in our content sharing system, if \u201cNotifications\u201d is not working, the site should gracefully handle that failure by removing the functionality instead of taking the whole site down. Swimlane is one of the commonly used fault isolation methodology. Swimlane adds a barrier to the service from other services so that failure on either of them won\u2019t affect the other. Say we roll out a new feature \u2018Advertisement\u2019 in our content sharing app. We can have two architectures If Ads are generated on the fly synchronously during each Newsfeed request, the faults in Ads feature gets propagated to Newsfeed feature. Instead if we swimlane \u201cGeneration of Ads\u201d service and use a shared storage to populate Newsfeed App, Ads failures won\u2019t cascade to Newsfeed and worst case if Ads don\u2019t meet SLA , we can have Newsfeed without Ads. Let's take another example, we come up with a new model for our Content sharing App. Here we roll out enterprise content sharing App where enterprises pay for the service and the content should never be shared outside the enterprise. Swimlane Principles Principle 1: Nothing is shared (also known as \u201cshare as little as possible\u201d). The less that is shared within a swim lane, the more fault isolative the swim lane becomes. (as shown in Enterprise usecase) Principle 2: Nothing crosses a swim lane boundary. Synchronous (defined by expecting a request\u2014not the transfer protocol) communication never crosses a swim lane boundary; if it does, the boundary is drawn incorrectly. (as shown in Ads feature) Swimlane Approaches Approach 1: Swim lane the money-maker. Never allow your cash register to be compromised by other systems. (Tier 1 vs Tier 2 in enterprise use case) Approach 2: Swim lane the biggest sources of incidents. Identify the recurring causes of pain and isolate them.(if Ads feature is in code yellow, swim laning it is the best option) Approach 3: Swim lane natural barriers. Customer boundaries make good swim lanes.(Public vs Enterprise customers) Refer https://learning.oreilly.com/library/view/the-art-of/9780134031408/ch21.html#ch21 SRE Use cases: Work with the DC tech or cloud team to distribute infrastructure such that its immune to switch or power failures by creating fault zones within a Data Center https://docs.microsoft.com/en-us/azure/virtual-machines/manage-availability#use-availability-zones-to-protect-from-datacenter-level-failures Work with the partners and design interaction between services such that one service breakdown is not amplified in a cascading fashion to all upstreams","title":"Fault Tolerance"},{"location":"systems_design/fault-tolerance/#fault-tolerance","text":"Failures are not avoidable in any system and will happen all the time, hence we need to build systems that can tolerate failures or recover from them. In systems, failure is the norm rather than the exception. \"Anything that can go wrong will go wrong\u201d -- Murphy\u2019s Law \u201cComplex systems contain changing mixtures of failures latent within them\u201d -- How Complex Systems Fail.","title":"Fault Tolerance"},{"location":"systems_design/fault-tolerance/#fault-tolerance-failure-metrics","text":"Common failure metrics that get measured and tracked for any system. Mean time to repair (MTTR): The average time to repair and restore a failed system. Mean time between failures (MTBF): The average operational time between one device failure or system breakdown and the next. Mean time to failure (MTTF): The average time a device or system is expected to function before it fails. Mean time to detect (MTTD): The average time between the onset of a problem and when the organization detects it. Mean time to investigate (MTTI): The average time between the detection of an incident and when the organization begins to investigate its cause and solution. Mean time to restore service (MTRS): The average elapsed time from the detection of an incident until the affected system or component is again available to users. Mean time between system incidents (MTBSI): The average elapsed time between the detection of two consecutive incidents. MTBSI can be calculated by adding MTBF and MTRS (MTBSI = MTBF + MTRS). Failure rate: Another reliability metric, which measures the frequency with which a component or system fails. It is expressed as a number of failures over a unit of time.","title":"Fault Tolerance - Failure Metrics"},{"location":"systems_design/fault-tolerance/#refer","text":"https://www.splunk.com/en_us/data-insider/what-is-mean-time-to-repair.html","title":"Refer"},{"location":"systems_design/fault-tolerance/#fault-tolerance-fault-isolation-terms","text":"Systems should have a short circuit. Say in our content sharing system, if \u201cNotifications\u201d is not working, the site should gracefully handle that failure by removing the functionality instead of taking the whole site down. Swimlane is one of the commonly used fault isolation methodology. Swimlane adds a barrier to the service from other services so that failure on either of them won\u2019t affect the other. Say we roll out a new feature \u2018Advertisement\u2019 in our content sharing app. We can have two architectures If Ads are generated on the fly synchronously during each Newsfeed request, the faults in Ads feature gets propagated to Newsfeed feature. Instead if we swimlane \u201cGeneration of Ads\u201d service and use a shared storage to populate Newsfeed App, Ads failures won\u2019t cascade to Newsfeed and worst case if Ads don\u2019t meet SLA , we can have Newsfeed without Ads. Let's take another example, we come up with a new model for our Content sharing App. Here we roll out enterprise content sharing App where enterprises pay for the service and the content should never be shared outside the enterprise.","title":"Fault Tolerance - Fault Isolation Terms"},{"location":"systems_design/fault-tolerance/#swimlane-principles","text":"Principle 1: Nothing is shared (also known as \u201cshare as little as possible\u201d). The less that is shared within a swim lane, the more fault isolative the swim lane becomes. (as shown in Enterprise usecase) Principle 2: Nothing crosses a swim lane boundary. Synchronous (defined by expecting a request\u2014not the transfer protocol) communication never crosses a swim lane boundary; if it does, the boundary is drawn incorrectly. (as shown in Ads feature)","title":"Swimlane Principles"},{"location":"systems_design/fault-tolerance/#swimlane-approaches","text":"Approach 1: Swim lane the money-maker. Never allow your cash register to be compromised by other systems. (Tier 1 vs Tier 2 in enterprise use case) Approach 2: Swim lane the biggest sources of incidents. Identify the recurring causes of pain and isolate them.(if Ads feature is in code yellow, swim laning it is the best option) Approach 3: Swim lane natural barriers. Customer boundaries make good swim lanes.(Public vs Enterprise customers)","title":"Swimlane Approaches"},{"location":"systems_design/fault-tolerance/#refer_1","text":"https://learning.oreilly.com/library/view/the-art-of/9780134031408/ch21.html#ch21","title":"Refer"},{"location":"systems_design/fault-tolerance/#sre-use-cases","text":"Work with the DC tech or cloud team to distribute infrastructure such that its immune to switch or power failures by creating fault zones within a Data Center https://docs.microsoft.com/en-us/azure/virtual-machines/manage-availability#use-availability-zones-to-protect-from-datacenter-level-failures Work with the partners and design interaction between services such that one service breakdown is not amplified in a cascading fashion to all upstreams","title":"SRE Use cases:"},{"location":"systems_design/intro/","text":"Systems Design Pre - Requisites Fundamentals of common software system components: - Operating Systems - Networking - Databases RDBMS/NoSQL What to expect from this training Thinking about and designing for scalability, availability, and reliability of large scale software systems. What is not covered under this training Individual software components\u2019 scalability and reliability concerns like e.g. Databases, while the same scalability principles and thinking can be applied, these individual components have their own specific nuances when scaling them and thinking about their reliability. More light will be shed on concepts rather than on setting up and configuring components like Loadbalancers to achieve scalability, availability and reliability of systems Training Content Introduction Scalability High Availability Fault Tolerance Introduction So, how do you go about learning to design a system? \u201d Like most great questions, it showed a level of naivety that was breathtaking. The only short answer I could give was, essentially, that you learned how to design a system by designing systems and finding out what works and what doesn\u2019t work.\u201d Jim Waldo, Sun Microsystems, On System Design As software and hardware systems have multiple moving parts, we need to think about how those parts will grow, their failure modes, their inter-dependencies, how it will impact the users and the business. There is no one-shot method or way to learn or do system design, we only learn to design systems by designing and iterating on them. This course will be a starter to make one think about scalability, availability, and fault tolerance during systems design. Backstory Let\u2019s design a simple content sharing application where users can share photos, media in our application which can be liked by their friends. Let\u2019s start with a simple design of the application and evolve it as we learn system design concepts","title":"Intro"},{"location":"systems_design/intro/#systems-design","text":"","title":"Systems Design"},{"location":"systems_design/intro/#pre-requisites","text":"Fundamentals of common software system components: - Operating Systems - Networking - Databases RDBMS/NoSQL","title":"Pre - Requisites"},{"location":"systems_design/intro/#what-to-expect-from-this-training","text":"Thinking about and designing for scalability, availability, and reliability of large scale software systems.","title":"What to expect from this training"},{"location":"systems_design/intro/#what-is-not-covered-under-this-training","text":"Individual software components\u2019 scalability and reliability concerns like e.g. Databases, while the same scalability principles and thinking can be applied, these individual components have their own specific nuances when scaling them and thinking about their reliability. More light will be shed on concepts rather than on setting up and configuring components like Loadbalancers to achieve scalability, availability and reliability of systems","title":"What is not covered under this training"},{"location":"systems_design/intro/#training-content","text":"Introduction Scalability High Availability Fault Tolerance","title":"Training Content"},{"location":"systems_design/intro/#introduction","text":"So, how do you go about learning to design a system? \u201d Like most great questions, it showed a level of naivety that was breathtaking. The only short answer I could give was, essentially, that you learned how to design a system by designing systems and finding out what works and what doesn\u2019t work.\u201d Jim Waldo, Sun Microsystems, On System Design As software and hardware systems have multiple moving parts, we need to think about how those parts will grow, their failure modes, their inter-dependencies, how it will impact the users and the business. There is no one-shot method or way to learn or do system design, we only learn to design systems by designing and iterating on them. This course will be a starter to make one think about scalability, availability, and fault tolerance during systems design.","title":"Introduction"},{"location":"systems_design/intro/#backstory","text":"Let\u2019s design a simple content sharing application where users can share photos, media in our application which can be liked by their friends. Let\u2019s start with a simple design of the application and evolve it as we learn system design concepts","title":"Backstory"},{"location":"systems_design/scalability/","text":"Scalability What does scalability mean for a system/service? A system is composed of services/components, each service/component scalability needs to be tackled separately, and the scalability of the system as a whole. A service is said to be scalable if, as resources are added to the system, it results in increased performance in a manner proportional to resources added An always-on service is said to be scalable if adding resources to facilitate redundancy does not result in a loss of performance Refer https://www.allthingsdistributed.com/2006/03/a_word_on_scalability.html Scalability - AKF Scale Cube The Scale Cube is a model for segmenting services, defining microservices, and scaling products. It also creates a common language for teams to discuss scale related options in designing solutions. Following section talks about certain scaling patterns based on our inferences from AKF cube Scalability - Horizontal scaling Horizontal scaling stands for cloning of an application or service such that work can easily be distributed across instances with absolutely no bias. Lets see how our monolithic application improves with this principle Here DB is scaled separately from the application. This is to let you know each component\u2019s scaling capabilities can be different. Usually web applications can be scaled by adding resources unless there is no state stored inside the application. But DBs can be scaled only for Reads by adding more followers but Writes have to go to only one master to make sure data is consistent. There are some DBs which support multi master writes but we are keeping them out of scope at this point. Apps should be able to differentiate between Read and Writes to choose appropriate DB servers. Load balancers can split traffic between identical servers transparently. WHAT: Duplication of services or databases to spread transaction load. WHEN TO USE: Databases with a very high read-to-write ratio (5:1 or greater\u2014the higher the better). Because only read replicas of DBs can be scaled, not the Master. HOW TO USE: Simply clone services and implement a load balancer. For databases, ensure that the accessing code understands the difference between a read and a write. WHY: Allows for fast scale of transactions at the cost of duplicated data and functionality. KEY TAKEAWAYS: This is fast to implement, is low cost from a developer effort perspective, and can scale transaction volumes nicely. However, they tend to be high cost from the perspective of the operational cost of data. Cost here means if we have 3 followers and 1 Master DB, the same database will be stored as 4 copies in the 4 servers. Hence added storage cost Refer https://learning.oreilly.com/library/view/the-art-of/9780134031408/ch23.html Scalability Pattern - Load Balancing Improves the distribution of workloads across multiple computing resources, such as computers, a computer cluster, network links, central processing units, or disk drives. Commonly used technique is load balancing traffic across identical server clusters. Similar philosophy is used to load balance traffic across network links by ECMP , disk drives by RAID etc Aims to optimize resource use, maximize throughput, minimize response time, and avoid overload of any single resource. Using multiple components with load balancing instead of a single component may increase reliability and availability through redundancy. In our updated architecture diagram we have 4 servers to handle app traffic instead of a single server The device or system that performs load balancing is called a load balancer, abbreviated as LB. Refer https://en.wikipedia.org/wiki/Load_balancing_(computing) https://blog.envoyproxy.io/introduction-to-modern-network-load-balancing-and-proxying-a57f6ff80236 https://learning.oreilly.com/library/view/load-balancing-in/9781492038009/ https://learning.oreilly.com/library/view/practical-load-balancing/9781430236801/ http://shop.oreilly.com/product/9780596000509.do Scalability Pattern - LB Tasks What does an LB do? Service discovery: What backends are available in the system? In our architecture, 4 servers are available to serve App traffic. LB acts as a single endpoint that clients can use transparently to reach one of the 4 servers. Health checking: What backends are currently healthy and available to accept requests? If one out of the 4 App servers turns bad, LB should automatically short circuit the path so that clients don\u2019t sense any application downtime Load balancing: What algorithm should be used to balance individual requests across the healthy backends? There are many algorithms to distribute traffic across one of the four servers. Based on observations/experience, SRE can pick the algorithm that suits their pattern Scalability Pattern - LB Methods Common Load Balancing Methods Least Connection Method directs traffic to the server with the fewest active connections. Most useful when there are a large number of persistent connections in the traffic unevenly distributed between the servers. Works if clients maintain long lived connections Least Response Time Method directs traffic to the server with the fewest active connections and the lowest average response time. Here response time is used to provide feedback of server\u2019s health Round Robin Method rotates servers by directing traffic to the first available server and then moves that server to the bottom of the queue. Most useful when servers are of equal specification and there are not many persistent connections. IP Hash the IP address of the client determines which server receives the request. This can sometimes cause skewness in distribution but is useful if apps store some state locally and need some stickiness More advanced client/server-side example techniques - https://docs.nginx.com/nginx/admin-guide/load-balancer/ - http://cbonte.github.io/haproxy-dconv/2.2/intro.html#3.3.5 - https://twitter.github.io/finagle/guide/Clients.html#load-balancing Scalability Pattern - Caching - Content Delivery Networks (CDN) CDNs are added closer to the client\u2019s location. If the app has static data like images, Javascript, CSS which don\u2019t change very often, they can be cached. Since our example is a content sharing site, static content can be cached in CDNs with a suitable expiry. WHAT: Use CDNs (content delivery networks) to offload traffic from your site. WHEN TO USE: When speed improvements and scale warrant the additional cost. HOW TO USE: Most CDNs leverage DNS to serve content on your site\u2019s behalf. Thus you may need to make minor DNS changes or additions and move content to be served from new subdomains. Eg media-exp1.licdn.com is a domain used by Linkedin to serve static content Here a CNAME points the domain to the DNS of CDN provider dig media-exp1.licdn.com +short 2-01-2c3e-005c.cdx.cedexis.net. WHY: CDNs help offload traffic spikes and are often economical ways to scale parts of a site\u2019s traffic. They also often substantially improve page download times. KEY TAKEAWAYS: CDNs are a fast and simple way to offset the spikiness of traffic as well as traffic growth in general. Make sure you perform a cost-benefit analysis and monitor the CDN usage. If CDNs have a lot of cache misses, then we don\u2019t gain much from CDN and are still serving requests using our compute resources. Scalability - Microservices This pattern represents the separation of work by service or function within the application. Microservices are meant to address the issues associated with growth and complexity in the code base and data sets. The intent is to create fault isolation as well as to reduce response times. Microservices can scale transactions, data sizes, and codebase sizes. They are most effective in scaling the size and complexity of your codebase. They tend to cost a bit more than horizontal scaling because the engineering team needs to rewrite services or, at the very least, disaggregate them from the original monolithic application. WHAT: Sometimes referred to as scale through services or resources, this rule focuses on scaling by splitting data sets, transactions, and engineering teams along verb (services) or noun (resources) boundaries. WHEN TO USE: Very large data sets where relations between data are not necessary. Large, complex systems where scaling engineering resources requires specialization. HOW TO USE: Split up actions by using verbs, or resources by using nouns, or use a mix. Split both the services and the data along the lines defined by the verb/noun approach. WHY: Allows for efficient scaling of not only transactions but also very large data sets associated with those transactions. It also allows for the efficient scaling of teams. KEY TAKEAWAYS: Microservices allow for efficient scaling of transactions, large data sets, and can help with fault isolation. It helps reduce the communication overhead of teams. The codebase becomes less complex as disjoint features are decoupled and spun as new services thereby letting each service scale independently specific to its requirement. Refer https://learning.oreilly.com/library/view/the-art-of/9780134031408/ch23.html Scalability - Sharding This pattern represents the separation of work based on attributes that are looked up or determined at the time of the transaction. Most often, these are implemented as splits by requestor, customer, or client. Very often, a lookup service or deterministic algorithm will need to be written for these types of splits. Sharding aids in scaling transaction growth, scaling instruction sets, and decreasing processing time (the last by limiting the data necessary to perform any transaction). This is more effective at scaling growth in customers or clients. It can aid with disaster recovery efforts, and limit the impact of incidents to only a specific segment of customers. Here the auth data is sharded based on user names so that DBs can respond faster as the amount of data DBs have to work on has drastically reduced during queries. There can be other ways to split Here the whole data centre is split and replicated and clients are directed to a data centre based on their geography. This helps in improving performance as clients are directed to the closest Data centre and performance increases as we add more data centres. There are some replication and consistency overhead with this approach one needs to be aware of. This also gives fault tolerance by rolling out test features to one site and rollback if there is an impact to that geography WHAT: This is very often a split by some unique aspect of the customer such as customer ID, name, geography, and so on. WHEN TO USE: Very large, similar data sets such as large and rapidly growing customer bases or when the response time for a geographically distributed customer base is important. HOW TO USE: Identify something you know about the customer, such as customer ID, last name, geography, or device, and split or partition both data and services based on that attribute. WHY: Rapid customer growth exceeds other forms of data growth, or you have the need to perform fault isolation between certain customer groups as you scale. KEY TAKEAWAYS: Shards are effective at helping you to scale customer bases but can also be applied to other very large data sets that can\u2019t be pulled apart using the microservices methodology. Refer https://learning.oreilly.com/library/view/the-art-of/9780134031408/ch23.html SRE Use cases SREs in coordination with the network team work on how to map users traffic to a particular site. https://engineering.linkedin.com/blog/2017/05/trafficshift--load-testing-at-scale SREs work closely with the Dev team to split monoliths to multiple microservices that are easy to run and manage SREs work on improving Load Balancers' reliability, service discovery and performance SREs work closely to split Data into shards and manage data integrity and consistency. https://engineering.linkedin.com/espresso/introducing-espresso-linkedins-hot-new-distributed-document-store SREs work to set up, configure and improve CDN cache hit rate.","title":"Scalability"},{"location":"systems_design/scalability/#scalability","text":"What does scalability mean for a system/service? A system is composed of services/components, each service/component scalability needs to be tackled separately, and the scalability of the system as a whole. A service is said to be scalable if, as resources are added to the system, it results in increased performance in a manner proportional to resources added An always-on service is said to be scalable if adding resources to facilitate redundancy does not result in a loss of performance","title":"Scalability"},{"location":"systems_design/scalability/#refer","text":"https://www.allthingsdistributed.com/2006/03/a_word_on_scalability.html","title":"Refer"},{"location":"systems_design/scalability/#scalability-akf-scale-cube","text":"The Scale Cube is a model for segmenting services, defining microservices, and scaling products. It also creates a common language for teams to discuss scale related options in designing solutions. Following section talks about certain scaling patterns based on our inferences from AKF cube","title":"Scalability - AKF Scale Cube"},{"location":"systems_design/scalability/#scalability-horizontal-scaling","text":"Horizontal scaling stands for cloning of an application or service such that work can easily be distributed across instances with absolutely no bias. Lets see how our monolithic application improves with this principle Here DB is scaled separately from the application. This is to let you know each component\u2019s scaling capabilities can be different. Usually web applications can be scaled by adding resources unless there is no state stored inside the application. But DBs can be scaled only for Reads by adding more followers but Writes have to go to only one master to make sure data is consistent. There are some DBs which support multi master writes but we are keeping them out of scope at this point. Apps should be able to differentiate between Read and Writes to choose appropriate DB servers. Load balancers can split traffic between identical servers transparently. WHAT: Duplication of services or databases to spread transaction load. WHEN TO USE: Databases with a very high read-to-write ratio (5:1 or greater\u2014the higher the better). Because only read replicas of DBs can be scaled, not the Master. HOW TO USE: Simply clone services and implement a load balancer. For databases, ensure that the accessing code understands the difference between a read and a write. WHY: Allows for fast scale of transactions at the cost of duplicated data and functionality. KEY TAKEAWAYS: This is fast to implement, is low cost from a developer effort perspective, and can scale transaction volumes nicely. However, they tend to be high cost from the perspective of the operational cost of data. Cost here means if we have 3 followers and 1 Master DB, the same database will be stored as 4 copies in the 4 servers. Hence added storage cost","title":"Scalability - Horizontal scaling"},{"location":"systems_design/scalability/#refer_1","text":"https://learning.oreilly.com/library/view/the-art-of/9780134031408/ch23.html","title":"Refer"},{"location":"systems_design/scalability/#scalability-pattern-load-balancing","text":"Improves the distribution of workloads across multiple computing resources, such as computers, a computer cluster, network links, central processing units, or disk drives. Commonly used technique is load balancing traffic across identical server clusters. Similar philosophy is used to load balance traffic across network links by ECMP , disk drives by RAID etc Aims to optimize resource use, maximize throughput, minimize response time, and avoid overload of any single resource. Using multiple components with load balancing instead of a single component may increase reliability and availability through redundancy. In our updated architecture diagram we have 4 servers to handle app traffic instead of a single server The device or system that performs load balancing is called a load balancer, abbreviated as LB.","title":"Scalability Pattern - Load Balancing"},{"location":"systems_design/scalability/#refer_2","text":"https://en.wikipedia.org/wiki/Load_balancing_(computing) https://blog.envoyproxy.io/introduction-to-modern-network-load-balancing-and-proxying-a57f6ff80236 https://learning.oreilly.com/library/view/load-balancing-in/9781492038009/ https://learning.oreilly.com/library/view/practical-load-balancing/9781430236801/ http://shop.oreilly.com/product/9780596000509.do","title":"Refer"},{"location":"systems_design/scalability/#scalability-pattern-lb-tasks","text":"What does an LB do?","title":"Scalability Pattern - LB Tasks"},{"location":"systems_design/scalability/#service-discovery","text":"What backends are available in the system? In our architecture, 4 servers are available to serve App traffic. LB acts as a single endpoint that clients can use transparently to reach one of the 4 servers.","title":"Service discovery:"},{"location":"systems_design/scalability/#health-checking","text":"What backends are currently healthy and available to accept requests? If one out of the 4 App servers turns bad, LB should automatically short circuit the path so that clients don\u2019t sense any application downtime","title":"Health checking:"},{"location":"systems_design/scalability/#load-balancing","text":"What algorithm should be used to balance individual requests across the healthy backends? There are many algorithms to distribute traffic across one of the four servers. Based on observations/experience, SRE can pick the algorithm that suits their pattern","title":"Load balancing:"},{"location":"systems_design/scalability/#scalability-pattern-lb-methods","text":"Common Load Balancing Methods","title":"Scalability Pattern - LB Methods"},{"location":"systems_design/scalability/#least-connection-method","text":"directs traffic to the server with the fewest active connections. Most useful when there are a large number of persistent connections in the traffic unevenly distributed between the servers. Works if clients maintain long lived connections","title":"Least Connection Method"},{"location":"systems_design/scalability/#least-response-time-method","text":"directs traffic to the server with the fewest active connections and the lowest average response time. Here response time is used to provide feedback of server\u2019s health","title":"Least Response Time Method"},{"location":"systems_design/scalability/#round-robin-method","text":"rotates servers by directing traffic to the first available server and then moves that server to the bottom of the queue. Most useful when servers are of equal specification and there are not many persistent connections.","title":"Round Robin Method"},{"location":"systems_design/scalability/#ip-hash","text":"the IP address of the client determines which server receives the request. This can sometimes cause skewness in distribution but is useful if apps store some state locally and need some stickiness More advanced client/server-side example techniques - https://docs.nginx.com/nginx/admin-guide/load-balancer/ - http://cbonte.github.io/haproxy-dconv/2.2/intro.html#3.3.5 - https://twitter.github.io/finagle/guide/Clients.html#load-balancing","title":"IP Hash"},{"location":"systems_design/scalability/#scalability-pattern-caching-content-delivery-networks-cdn","text":"CDNs are added closer to the client\u2019s location. If the app has static data like images, Javascript, CSS which don\u2019t change very often, they can be cached. Since our example is a content sharing site, static content can be cached in CDNs with a suitable expiry. WHAT: Use CDNs (content delivery networks) to offload traffic from your site. WHEN TO USE: When speed improvements and scale warrant the additional cost. HOW TO USE: Most CDNs leverage DNS to serve content on your site\u2019s behalf. Thus you may need to make minor DNS changes or additions and move content to be served from new subdomains. Eg media-exp1.licdn.com is a domain used by Linkedin to serve static content Here a CNAME points the domain to the DNS of CDN provider dig media-exp1.licdn.com +short 2-01-2c3e-005c.cdx.cedexis.net. WHY: CDNs help offload traffic spikes and are often economical ways to scale parts of a site\u2019s traffic. They also often substantially improve page download times. KEY TAKEAWAYS: CDNs are a fast and simple way to offset the spikiness of traffic as well as traffic growth in general. Make sure you perform a cost-benefit analysis and monitor the CDN usage. If CDNs have a lot of cache misses, then we don\u2019t gain much from CDN and are still serving requests using our compute resources.","title":"Scalability Pattern - Caching - Content Delivery Networks (CDN)"},{"location":"systems_design/scalability/#scalability-microservices","text":"This pattern represents the separation of work by service or function within the application. Microservices are meant to address the issues associated with growth and complexity in the code base and data sets. The intent is to create fault isolation as well as to reduce response times. Microservices can scale transactions, data sizes, and codebase sizes. They are most effective in scaling the size and complexity of your codebase. They tend to cost a bit more than horizontal scaling because the engineering team needs to rewrite services or, at the very least, disaggregate them from the original monolithic application. WHAT: Sometimes referred to as scale through services or resources, this rule focuses on scaling by splitting data sets, transactions, and engineering teams along verb (services) or noun (resources) boundaries. WHEN TO USE: Very large data sets where relations between data are not necessary. Large, complex systems where scaling engineering resources requires specialization. HOW TO USE: Split up actions by using verbs, or resources by using nouns, or use a mix. Split both the services and the data along the lines defined by the verb/noun approach. WHY: Allows for efficient scaling of not only transactions but also very large data sets associated with those transactions. It also allows for the efficient scaling of teams. KEY TAKEAWAYS: Microservices allow for efficient scaling of transactions, large data sets, and can help with fault isolation. It helps reduce the communication overhead of teams. The codebase becomes less complex as disjoint features are decoupled and spun as new services thereby letting each service scale independently specific to its requirement.","title":"Scalability - Microservices"},{"location":"systems_design/scalability/#refer_3","text":"https://learning.oreilly.com/library/view/the-art-of/9780134031408/ch23.html","title":"Refer"},{"location":"systems_design/scalability/#scalability-sharding","text":"This pattern represents the separation of work based on attributes that are looked up or determined at the time of the transaction. Most often, these are implemented as splits by requestor, customer, or client. Very often, a lookup service or deterministic algorithm will need to be written for these types of splits. Sharding aids in scaling transaction growth, scaling instruction sets, and decreasing processing time (the last by limiting the data necessary to perform any transaction). This is more effective at scaling growth in customers or clients. It can aid with disaster recovery efforts, and limit the impact of incidents to only a specific segment of customers. Here the auth data is sharded based on user names so that DBs can respond faster as the amount of data DBs have to work on has drastically reduced during queries. There can be other ways to split Here the whole data centre is split and replicated and clients are directed to a data centre based on their geography. This helps in improving performance as clients are directed to the closest Data centre and performance increases as we add more data centres. There are some replication and consistency overhead with this approach one needs to be aware of. This also gives fault tolerance by rolling out test features to one site and rollback if there is an impact to that geography WHAT: This is very often a split by some unique aspect of the customer such as customer ID, name, geography, and so on. WHEN TO USE: Very large, similar data sets such as large and rapidly growing customer bases or when the response time for a geographically distributed customer base is important. HOW TO USE: Identify something you know about the customer, such as customer ID, last name, geography, or device, and split or partition both data and services based on that attribute. WHY: Rapid customer growth exceeds other forms of data growth, or you have the need to perform fault isolation between certain customer groups as you scale. KEY TAKEAWAYS: Shards are effective at helping you to scale customer bases but can also be applied to other very large data sets that can\u2019t be pulled apart using the microservices methodology.","title":"Scalability - Sharding"},{"location":"systems_design/scalability/#refer_4","text":"https://learning.oreilly.com/library/view/the-art-of/9780134031408/ch23.html","title":"Refer"},{"location":"systems_design/scalability/#sre-use-cases","text":"SREs in coordination with the network team work on how to map users traffic to a particular site. https://engineering.linkedin.com/blog/2017/05/trafficshift--load-testing-at-scale SREs work closely with the Dev team to split monoliths to multiple microservices that are easy to run and manage SREs work on improving Load Balancers' reliability, service discovery and performance SREs work closely to split Data into shards and manage data integrity and consistency. https://engineering.linkedin.com/espresso/introducing-espresso-linkedins-hot-new-distributed-document-store SREs work to set up, configure and improve CDN cache hit rate.","title":"SRE Use cases"}]}
\ No newline at end of file
diff --git a/sitemap.xml b/sitemap.xml
index d63f781..9dae467 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -2,32 +2,72 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
     <url>
      <loc>None</loc>
-     <lastmod>2020-11-02</lastmod>
+     <lastmod>2020-11-03</lastmod>
      <changefreq>daily</changefreq>
     </url>
     <url>
      <loc>None</loc>
-     <lastmod>2020-11-02</lastmod>
+     <lastmod>2020-11-03</lastmod>
      <changefreq>daily</changefreq>
     </url>
     <url>
      <loc>None</loc>
-     <lastmod>2020-11-02</lastmod>
+     <lastmod>2020-11-03</lastmod>
      <changefreq>daily</changefreq>
     </url>
     <url>
      <loc>None</loc>
-     <lastmod>2020-11-02</lastmod>
+     <lastmod>2020-11-03</lastmod>
      <changefreq>daily</changefreq>
     </url>
     <url>
      <loc>None</loc>
-     <lastmod>2020-11-02</lastmod>
+     <lastmod>2020-11-03</lastmod>
      <changefreq>daily</changefreq>
     </url>
     <url>
      <loc>None</loc>
-     <lastmod>2020-11-02</lastmod>
+     <lastmod>2020-11-03</lastmod>
+     <changefreq>daily</changefreq>
+    </url>
+    <url>
+     <loc>None</loc>
+     <lastmod>2020-11-03</lastmod>
+     <changefreq>daily</changefreq>
+    </url>
+    <url>
+     <loc>None</loc>
+     <lastmod>2020-11-03</lastmod>
+     <changefreq>daily</changefreq>
+    </url>
+    <url>
+     <loc>None</loc>
+     <lastmod>2020-11-03</lastmod>
+     <changefreq>daily</changefreq>
+    </url>
+    <url>
+     <loc>None</loc>
+     <lastmod>2020-11-03</lastmod>
+     <changefreq>daily</changefreq>
+    </url>
+    <url>
+     <loc>None</loc>
+     <lastmod>2020-11-03</lastmod>
+     <changefreq>daily</changefreq>
+    </url>
+    <url>
+     <loc>None</loc>
+     <lastmod>2020-11-03</lastmod>
+     <changefreq>daily</changefreq>
+    </url>
+    <url>
+     <loc>None</loc>
+     <lastmod>2020-11-03</lastmod>
+     <changefreq>daily</changefreq>
+    </url>
+    <url>
+     <loc>None</loc>
+     <lastmod>2020-11-03</lastmod>
      <changefreq>daily</changefreq>
     </url>
 </urlset>
\ No newline at end of file
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index 54f6e8095ce128e8688ecdfdeb18d5ea0daad9a8..b29426d20d6ba5e0cd079b28267e95c47c1a147e 100644
GIT binary patch
literal 204
zcmV;-05ks|iwFn~U7=qB|8r?{Wo=<_E_iKh0PU4Q4#FT1MfW`gVPBwiqlVI*OILaT
zLa`-jp)zRo_EKAoS1@5V^XJdcyyfuvG4$xJJs4*zTo8tlw#qfewiP~~4*3?>BI7sT
zgAOQ$EmXJ*Vcb(XO%nkV^dLtd&bJg)7YCrLNg3H;k)c@gfkh$SJ0*_JYEEbMF(rh-
zHKJrC%Znmso48)RHKICcTYd5RDjI3}C#OHoLZM$w0w!PrCSU^o8SvfWuMsb|hRVvD
G1^@uJ?^((K

literal 198
zcmV;%06G63iwFob*`Hqm|8r?{Wo=<_E_iKh0PT{o5`r)chIgK#*PW&?saMM8=%f$O
zD;0VNEjCs3?Ja_ytGik9=g)unmWcX3^yFp&TJI`aFiwea+Bep<6+IpI`G(dq6W7qA
zi6n&`R<w(8+yP9}#Lz}Fs1cbDEuikwfUugt*_O(Tq?!*biiqB8dGO8%nAO`<7zf|T
zl9xO$ikz2ey?SdzcgnTq9Lz;F%JvTdU(P~-59Sa21AhX(TKaMM1lY7=g%bk+0Lz(S
AUjP6A

diff --git a/systems_design/availability/index.html b/systems_design/availability/index.html
index fb7cece..34f027a 100644
--- a/systems_design/availability/index.html
+++ b/systems_design/availability/index.html
@@ -49,6 +49,48 @@
                             <li >
                                 <a href="../..">Home</a>
                             </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Git <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../git/git-basics/">Git Basics</a>
+</li>
+                                    
+<li >
+    <a href="../../git/branches/">Working With Branches</a>
+</li>
+                                    
+<li >
+    <a href="../../git/github-hooks/">Github and Hooks</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Python and Web <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../python_web/intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/python-concepts/">Some Python Concepts</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/python-web-flask/">Python, Web and Flask</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/url-shorten-app/">The URL Shortening App</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/sre-conclusion/">SRE Aspects of The App and Conclusion</a>
+</li>
+                                </ul>
+                            </li>
                             <li class="dropdown active">
                                 <a href="#" class="dropdown-toggle" data-toggle="dropdown">Systems Design <b class="caret"></b></a>
                                 <ul class="dropdown-menu">
diff --git a/systems_design/conclusion/index.html b/systems_design/conclusion/index.html
index 160ef2c..768b9cb 100644
--- a/systems_design/conclusion/index.html
+++ b/systems_design/conclusion/index.html
@@ -49,6 +49,48 @@
                             <li >
                                 <a href="../..">Home</a>
                             </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Git <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../git/git-basics/">Git Basics</a>
+</li>
+                                    
+<li >
+    <a href="../../git/branches/">Working With Branches</a>
+</li>
+                                    
+<li >
+    <a href="../../git/github-hooks/">Github and Hooks</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Python and Web <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../python_web/intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/python-concepts/">Some Python Concepts</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/python-web-flask/">Python, Web and Flask</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/url-shorten-app/">The URL Shortening App</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/sre-conclusion/">SRE Aspects of The App and Conclusion</a>
+</li>
+                                </ul>
+                            </li>
                             <li class="dropdown active">
                                 <a href="#" class="dropdown-toggle" data-toggle="dropdown">Systems Design <b class="caret"></b></a>
                                 <ul class="dropdown-menu">
diff --git a/systems_design/fault-tolerance/index.html b/systems_design/fault-tolerance/index.html
index dfe4aa3..36a2dd9 100644
--- a/systems_design/fault-tolerance/index.html
+++ b/systems_design/fault-tolerance/index.html
@@ -49,6 +49,48 @@
                             <li >
                                 <a href="../..">Home</a>
                             </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Git <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../git/git-basics/">Git Basics</a>
+</li>
+                                    
+<li >
+    <a href="../../git/branches/">Working With Branches</a>
+</li>
+                                    
+<li >
+    <a href="../../git/github-hooks/">Github and Hooks</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Python and Web <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../python_web/intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/python-concepts/">Some Python Concepts</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/python-web-flask/">Python, Web and Flask</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/url-shorten-app/">The URL Shortening App</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/sre-conclusion/">SRE Aspects of The App and Conclusion</a>
+</li>
+                                </ul>
+                            </li>
                             <li class="dropdown active">
                                 <a href="#" class="dropdown-toggle" data-toggle="dropdown">Systems Design <b class="caret"></b></a>
                                 <ul class="dropdown-menu">
diff --git a/systems_design/intro/index.html b/systems_design/intro/index.html
index c4cddc6..5f8a0c5 100644
--- a/systems_design/intro/index.html
+++ b/systems_design/intro/index.html
@@ -49,6 +49,48 @@
                             <li >
                                 <a href="../..">Home</a>
                             </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Git <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../git/git-basics/">Git Basics</a>
+</li>
+                                    
+<li >
+    <a href="../../git/branches/">Working With Branches</a>
+</li>
+                                    
+<li >
+    <a href="../../git/github-hooks/">Github and Hooks</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Python and Web <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../python_web/intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/python-concepts/">Some Python Concepts</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/python-web-flask/">Python, Web and Flask</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/url-shorten-app/">The URL Shortening App</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/sre-conclusion/">SRE Aspects of The App and Conclusion</a>
+</li>
+                                </ul>
+                            </li>
                             <li class="dropdown active">
                                 <a href="#" class="dropdown-toggle" data-toggle="dropdown">Systems Design <b class="caret"></b></a>
                                 <ul class="dropdown-menu">
@@ -83,7 +125,7 @@
                             </a>
                         </li>
                             <li >
-                                <a rel="next" href="../..">
+                                <a rel="next" href="../../python_web/sre-conclusion/">
                                     <i class="fa fa-arrow-left"></i> Previous
                                 </a>
                             </li>
diff --git a/systems_design/scalability/index.html b/systems_design/scalability/index.html
index 158abbb..e4f337b 100644
--- a/systems_design/scalability/index.html
+++ b/systems_design/scalability/index.html
@@ -49,6 +49,48 @@
                             <li >
                                 <a href="../..">Home</a>
                             </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Git <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../git/git-basics/">Git Basics</a>
+</li>
+                                    
+<li >
+    <a href="../../git/branches/">Working With Branches</a>
+</li>
+                                    
+<li >
+    <a href="../../git/github-hooks/">Github and Hooks</a>
+</li>
+                                </ul>
+                            </li>
+                            <li class="dropdown">
+                                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Python and Web <b class="caret"></b></a>
+                                <ul class="dropdown-menu">
+                                    
+<li >
+    <a href="../../python_web/intro/">Intro</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/python-concepts/">Some Python Concepts</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/python-web-flask/">Python, Web and Flask</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/url-shorten-app/">The URL Shortening App</a>
+</li>
+                                    
+<li >
+    <a href="../../python_web/sre-conclusion/">SRE Aspects of The App and Conclusion</a>
+</li>
+                                </ul>
+                            </li>
                             <li class="dropdown active">
                                 <a href="#" class="dropdown-toggle" data-toggle="dropdown">Systems Design <b class="caret"></b></a>
                                 <ul class="dropdown-menu">
