index.html

<%= header_begin %>

<%= segment_script %>

  <title>Site Mapper - Algorithmia</title>
	<link rel="canonical" href="https://demos.algorithmia.com/<%= slug %>" />
	<meta name="description" content="Crawl a website, rank the pages, and extract content with Algorithmia's microservices.">

	<!-- FB -->
	<meta property="og:title" content="Crawl a Website and Rank Pages">
	<meta property="og:url" content="https://demos.algorithmia.com/sitemap/">
	<meta property="og:description" content="Crawl a website, rank the pages, and extract content with Algorithmia's microservices">
	<meta property="og:locale" content="en_US" />
	<meta property="og:type" content="website" />
	<meta property="og:image" content="https://demos.algorithmia.com/sitemap/public/images/fbshare.jpg" />
	<meta property="og:site_name" content="Algorithmia" />

	<!-- TWTR -->
	<meta name="twitter:card" content="summary_large_image" />
	<meta name="twitter:site" content="@algorithmia" />
	<meta name="twitter:title" content="Crawl a Website and Rank Pages" />
	<meta name="twitter:description" content="Crawl a website, rank the pages, and extract content with Algorithmia's microservices" />
	<meta name="twitter:image" content="https://demos.algorithmia.com/sitemap/public/images/twshare.jpg" />
	<meta name="twitter:creator" content="@algorithmia" />

	<!-- OTHER -->
	<meta itemprop="name" content="Crawl a Website and Rank Pages">
	<meta itemprop="description" content="Crawl a website, rank the pages, and extract content with Algorithmia's microservices">
<%= header_end %>

<section class="hero purple-grad-background">
  <div class="container">

    <div class="row row-sm">
      <div class="col-md-8">
        <h1>Site Mapper</h1>
        <h4 class="light whitespace-md">Look at Your Website the Way Google Does</h4>
      </div>
    </div>

    <div class="row">
      <div class="col-md-6">
        <div class="card">
        <!-- Inputs -->
          <form onsubmit="startScrape(); return false;">
            <div class="row whitespace-none">
              <div class="col-xs-12 col-sm-10 whitespace-sm">
                <label>Website to scrape</label>
                <input type="text" class="form-control url" id="siteUrl">
              </div>
              <div class="col-xs-4 col-sm-2 whitespace-sm depth-wrapper">
                <label title="Sets the maximum number of pages to crawl">Depth</label>
                <input type="number" min="1" max="5" class="form-control depth" id="depthLimit">
              </div>
            </div>
            <div class="row whitespace-none">
              <div class="col-md-12">
                  <button class="btn btn-primary btn-dots" type="submit" onclick="startScrape()">
                    <span> <!-- Add class no-viz onClick -->
                      <span class="glyphicon glyphicon-search" aria-hidden="true"></span>
                      Map Site
                    </span>
                      <div class="dots-container hidden">
                        <div class="dot dot1"></div>
                        <div class="dot dot2"></div>
                        <div class="dot dot3"></div>
                        <div class="dot dot4"></div>
                    </div>
                  </button>
                <span id="scrape-status"></span>
              </div>
            </div>
          </form>
        </div>
      </div>
    </div>
  </div>
</section>

<!-- RESULTS -->
<section id="results" class="container" style="display: none">
  <div class="row row-md"> <!-- tab row -->
    <div class="col-sm-7">
      <!-- Nav tabs -->
      <ul class="nav nav-tabs" role="tablist">
        <li role="presentation" class="active"><a href="#viz" aria-controls="viz" role="tab" data-toggle="tab">Sitemap Visualization</a></li>
        <li role="presentation"><a href="#pagerank-list" aria-controls="pagerank-list" role="tab" data-toggle="tab">Pagerank</a></li>
      </ul>
    </div>
  </div>
  <div class="row"> <!-- Containing row -->
    <div class="col-sm-7 tab-content">
    <!-- Tab pane -->
      <div role="tabpanel" class="tab-pane active" id="viz">
        <div class="row row-md">
          <div class="col-md-12">
              <svg id="viz-panel" class="viz">
                <g class="viewport">
                  <g class="bottom"></g>
                  <g class="middle"></g>
                  <g class="top"></g>
                </g>
              </svg>
          </div>
        </div>
      </div>
      <!-- Tab pane -->
      <div role="tabpanel" class="tab-pane" id="pagerank-list">
        <div class="row row-md">
          <div class="col-md-12">
            <div id="pagerank-panel">
              <div class="row whitespace-sm" id="pagerank-sorted"></div>
            </div>
          </div>
        </div>
      </div>
    </div> <!-- End tabbed content col -->

    <div class="col-sm-5" id="link-details" style="display: none">
      <p class="lg border-bottom">PAGE DATA</p>
      <p class="item-title">URL</p>
      <p><a id="link-url" target="_blank"></a></p>
      <p class="item-title">PAGERANK</p>
      <p id="link-rank"></p>
      <p class="item-title">GENERATED SUMMARY</p>
      <p id="link-summary"></p>
      <p class="item-title">GENERATED TAGS</p>
      <div class="tags" id="link-tags">
        <span class="aspinner demo-spinner"></span>
      </div>
    </div>
  </div> <!-- End containing row -->

</section>

<!-- EXPLANATION -->
<div class="grey-background border-bottom">
<section class="container">
  <div class="row row-md">
    <div class="col-md-12">
      <h2 class="whitespace-none">How it works</h2>
      <h4>Crawl a website, rank the pages, and extract content</h4>
    </div>
  </div>
  <div class="row row-md">
    <div class="col-md-8">
      <h3>Tools used</h3>
      <ul>
        <li><a href="https://algorithmia.com/algorithms/web/GetLinks">Get Links</a> – examines a webpage and returns links to other pages</li>
        <li><a href="https://algorithmia.com/algorithms/thatguy2048/PageRank">Page Rank</a> – a simple implementation of the <a href="https://en.wikipedia.org/wiki/PageRank" target="_blank">PageRank algorithm</a></li>
        <li><a href="https://algorithmia.com/algorithms/util/Url2Text">URL2Text</a> – extracts the main content from a webpage</li>
        <li><a href="https://algorithmia.com/algorithms/nlp/Summarizer">Summarizer</a> – creates a summary by extracting key topic sentences</li>
        <li><a href="https://algorithmia.com/algorithms/nlp/AutoTag">Auto-Tag</a> – generates keywords via Latent Dirichlet Allocation</li>
        <li><a href="https://d3js.org/" target="_blank">D3</a> - a JavaScript library for generating graphs and visualizations</li>
      </ul>
    </div>
  </div>
  <div class="row row-md">
    <div class="col-md-8">
      <h3>Source Data</h3>
      <p>We examine the provided URL, traversing the links found in that page and the pages linked to – up to a certain depth. We then generate a visual map and ranked list based on the links between those pages. For each page, we also provide a summary and list of tags.</p>
    </div>
  </div>
  <div class="row">
    <div class="col-md-8">
      <h3>Takeaway</h3>
      <p>Algorithmia's easy-to-use microservices make it possible to quickly traverse and examine a large number of pages without the need to set up any infrastructure. By combining these services with readily available visualization tools, a developer can rapidly assemble a site crawler without ever leaving the browser environment. </p>
    </div>
  </div>
</section>
</div>

  <!-- BUILT FOR DEVS -->
  <section class="container">
    <div class="row">
      <div class="col-md-9">
        <h2 class="whitespace-sm">Built For Developers</h2>
        <h4 class="whitespace-md">A simple, scalable API for machine intelligence</h4>
          <p class="item-title">SAMPLE INPUT</p>
          <pre class="whitespace-md"><code class="python hljs"><span class="hljs-keyword">import</span> Algorithmia
client = Algorithmia.client(<span class="hljs-string">'API KEY'</span>)
algo = client.algo(<span class="hljs-string">'web/GetLinks/0.1.5'</span>)
<span class="hljs-keyword">print</span> algo.pipe(<span class="hljs-string">"https://algorithmia.com"</span>)</code>
</pre>
            <p class="item-title">SAMPLE OUTPUT</p>
            <pre><code class="json hljs">[
  "http://developers.algorithmia.com",
  "https://algorithmia.com/terms",
  ...
]</code></pre>
        <a href="https://en.wikipedia.org/wiki/PageRank" class="btn btn-primary-inverse">
          LEARN MORE
        </a>
      </div>
    </div>
  </section>


  <!-- JS -->
  <script src="<%= slug %>/public/js/TweenMax.min.js"></script>
  <script src="<%= slug %>/public/js/dots.js"></script>
  <script src="<%= slug %>/public/js/d3.min.js"></script>
  <script src="<%= slug %>/public/js/algorithmia.d3.js"></script>

  <%= footer %>