1733 lines
210 KiB
HTML
1733 lines
210 KiB
HTML
|
|
|
|
<!DOCTYPE html>
|
|
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
|
|
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
|
|
<head>
|
|
<meta charset="utf-8">
|
|
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
|
|
<title>urllib2 — Binary Ninja API Documentation v1.1</title>
|
|
|
|
|
|
|
|
|
|
<link rel="shortcut icon" href="../_static/favicon.ico"/>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../_static/css/other.css" type="text/css" />
|
|
|
|
|
|
|
|
<link rel="index" title="Index"
|
|
href="../genindex.html"/>
|
|
<link rel="search" title="Search" href="../search.html"/>
|
|
<link rel="top" title="Binary Ninja API Documentation v1.1" href="../index.html"/>
|
|
<link rel="up" title="Module code" href="index.html"/>
|
|
|
|
|
|
<script src="../_static/js/modernizr.min.js"></script>
|
|
|
|
</head>
|
|
|
|
<body class="wy-body-for-nav" role="document">
|
|
|
|
|
|
<div class="wy-grid-for-nav">
|
|
|
|
|
|
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
|
<div class="wy-side-scroll">
|
|
<div class="wy-side-nav-search">
|
|
|
|
|
|
|
|
<a href="../index.html" class="icon icon-home"> Binary Ninja API
|
|
|
|
|
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
<div class="version">
|
|
<script>
|
|
window.addEventListener("load", function() {
|
|
var ver = document.getElementsByClassName("version");
|
|
ver[0].innerHTML = DOCUMENTATION_OPTIONS['VERSION'];
|
|
});
|
|
</script>
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<div role="search">
|
|
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
|
<input type="text" name="q" placeholder="Search docs" />
|
|
<input type="hidden" name="check_keywords" value="yes" />
|
|
<input type="hidden" name="area" value="default" />
|
|
</form>
|
|
</div>
|
|
|
|
|
|
</div>
|
|
|
|
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
|
|
|
|
|
|
|
|
<ul>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.architecture-module.html">architecture</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.atexit-module.html">atexit</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.basicblock-module.html">basicblock</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.binaryview-module.html">binaryview</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.callingconvention-module.html">callingconvention</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.databuffer-module.html">databuffer</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.datarender-module.html">datarender</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.demangle-module.html">demangle</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.downloadprovider-module.html">downloadprovider</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.enums-module.html">enums</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.fileaccessor-module.html">fileaccessor</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.filemetadata-module.html">filemetadata</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.flowgraph-module.html">flowgraph</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.function-module.html">function</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.functionrecognizer-module.html">functionrecognizer</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.highlight-module.html">highlight</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.interaction-module.html">interaction</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.lineardisassembly-module.html">lineardisassembly</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.log-module.html">log</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.lowlevelil-module.html">lowlevelil</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.mainthread-module.html">mainthread</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.mediumlevelil-module.html">mediumlevelil</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.metadata-module.html">metadata</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.numbers-module.html">numbers</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.platform-module.html">platform</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.plugin-module.html">plugin</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.pluginmanager-module.html">pluginmanager</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.scriptingprovider-module.html">scriptingprovider</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.settings-module.html">settings</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.transform-module.html">transform</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.types-module.html">types</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.undoaction-module.html">undoaction</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../binaryninja.update-module.html">update</a></li>
|
|
</ul>
|
|
|
|
|
|
|
|
</div>
|
|
</div>
|
|
</nav>
|
|
|
|
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
|
|
|
|
|
|
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
|
|
|
|
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
|
<a href="../index.html">Binary Ninja API</a>
|
|
|
|
</nav>
|
|
|
|
|
|
|
|
<div class="wy-nav-content">
|
|
<div class="rst-content">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<div role="navigation" aria-label="breadcrumbs navigation">
|
|
<ul class="wy-breadcrumbs">
|
|
|
|
<li><a href="../index.html">Docs</a> »</li>
|
|
|
|
<li><a href="index.html">Module code</a> »</li>
|
|
|
|
<li>urllib2</li>
|
|
|
|
|
|
<li class="wy-breadcrumbs-aside">
|
|
|
|
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
<hr/>
|
|
</div>
|
|
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
|
<div itemprop="articleBody">
|
|
|
|
<h1>Source code for urllib2</h1><div class="highlight"><pre>
|
|
<span></span><span class="sd">"""An extensible library for opening URLs using a variety of protocols</span>
|
|
|
|
<span class="sd">The simplest way to use this module is to call the urlopen function,</span>
|
|
<span class="sd">which accepts a string containing a URL or a Request object (described</span>
|
|
<span class="sd">below). It opens the URL and returns the results as file-like</span>
|
|
<span class="sd">object; the returned object has some extra methods described below.</span>
|
|
|
|
<span class="sd">The OpenerDirector manages a collection of Handler objects that do</span>
|
|
<span class="sd">all the actual work. Each Handler implements a particular protocol or</span>
|
|
<span class="sd">option. The OpenerDirector is a composite object that invokes the</span>
|
|
<span class="sd">Handlers needed to open the requested URL. For example, the</span>
|
|
<span class="sd">HTTPHandler performs HTTP GET and POST requests and deals with</span>
|
|
<span class="sd">non-error returns. The HTTPRedirectHandler automatically deals with</span>
|
|
<span class="sd">HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler</span>
|
|
<span class="sd">deals with digest authentication.</span>
|
|
|
|
<span class="sd">urlopen(url, data=None) -- Basic usage is the same as original</span>
|
|
<span class="sd">urllib. pass the url and optionally data to post to an HTTP URL, and</span>
|
|
<span class="sd">get a file-like object back. One difference is that you can also pass</span>
|
|
<span class="sd">a Request instance instead of URL. Raises a URLError (subclass of</span>
|
|
<span class="sd">IOError); for HTTP errors, raises an HTTPError, which can also be</span>
|
|
<span class="sd">treated as a valid response.</span>
|
|
|
|
<span class="sd">build_opener -- Function that creates a new OpenerDirector instance.</span>
|
|
<span class="sd">Will install the default handlers. Accepts one or more Handlers as</span>
|
|
<span class="sd">arguments, either instances or Handler classes that it will</span>
|
|
<span class="sd">instantiate. If one of the argument is a subclass of the default</span>
|
|
<span class="sd">handler, the argument will be installed instead of the default.</span>
|
|
|
|
<span class="sd">install_opener -- Installs a new opener as the default opener.</span>
|
|
|
|
<span class="sd">objects of interest:</span>
|
|
|
|
<span class="sd">OpenerDirector -- Sets up the User Agent as the Python-urllib client and manages</span>
|
|
<span class="sd">the Handler classes, while dealing with requests and responses.</span>
|
|
|
|
<span class="sd">Request -- An object that encapsulates the state of a request. The</span>
|
|
<span class="sd">state can be as simple as the URL. It can also include extra HTTP</span>
|
|
<span class="sd">headers, e.g. a User-Agent.</span>
|
|
|
|
<span class="sd">BaseHandler --</span>
|
|
|
|
<span class="sd">exceptions:</span>
|
|
<span class="sd">URLError -- A subclass of IOError, individual protocols have their own</span>
|
|
<span class="sd">specific subclass.</span>
|
|
|
|
<span class="sd">HTTPError -- Also a valid HTTP response, so you can treat an HTTP error</span>
|
|
<span class="sd">as an exceptional event or valid response.</span>
|
|
|
|
<span class="sd">internals:</span>
|
|
<span class="sd">BaseHandler and parent</span>
|
|
<span class="sd">_call_chain conventions</span>
|
|
|
|
<span class="sd">Example usage:</span>
|
|
|
|
<span class="sd">import urllib2</span>
|
|
|
|
<span class="sd"># set up authentication info</span>
|
|
<span class="sd">authinfo = urllib2.HTTPBasicAuthHandler()</span>
|
|
<span class="sd">authinfo.add_password(realm='PDQ Application',</span>
|
|
<span class="sd"> uri='https://mahler:8092/site-updates.py',</span>
|
|
<span class="sd"> user='klem',</span>
|
|
<span class="sd"> passwd='geheim$parole')</span>
|
|
|
|
<span class="sd">proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})</span>
|
|
|
|
<span class="sd"># build a new opener that adds authentication and caching FTP handlers</span>
|
|
<span class="sd">opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)</span>
|
|
|
|
<span class="sd"># install it</span>
|
|
<span class="sd">urllib2.install_opener(opener)</span>
|
|
|
|
<span class="sd">f = urllib2.urlopen('http://www.python.org/')</span>
|
|
|
|
|
|
<span class="sd">"""</span>
|
|
|
|
<span class="c1"># XXX issues:</span>
|
|
<span class="c1"># If an authentication error handler that tries to perform</span>
|
|
<span class="c1"># authentication for some reason but fails, how should the error be</span>
|
|
<span class="c1"># signalled? The client needs to know the HTTP error code. But if</span>
|
|
<span class="c1"># the handler knows that the problem was, e.g., that it didn't know</span>
|
|
<span class="c1"># that hash algo that requested in the challenge, it would be good to</span>
|
|
<span class="c1"># pass that information along to the client, too.</span>
|
|
<span class="c1"># ftp errors aren't handled cleanly</span>
|
|
<span class="c1"># check digest against correct (i.e. non-apache) implementation</span>
|
|
|
|
<span class="c1"># Possible extensions:</span>
|
|
<span class="c1"># complex proxies XXX not sure what exactly was meant by this</span>
|
|
<span class="c1"># abstract factory for opener</span>
|
|
|
|
<span class="kn">import</span> <span class="nn">base64</span>
|
|
<span class="kn">import</span> <span class="nn">hashlib</span>
|
|
<span class="kn">import</span> <span class="nn">httplib</span>
|
|
<span class="kn">import</span> <span class="nn">mimetools</span>
|
|
<span class="kn">import</span> <span class="nn">os</span>
|
|
<span class="kn">import</span> <span class="nn">posixpath</span>
|
|
<span class="kn">import</span> <span class="nn">random</span>
|
|
<span class="kn">import</span> <span class="nn">re</span>
|
|
<span class="kn">import</span> <span class="nn">socket</span>
|
|
<span class="kn">import</span> <span class="nn">sys</span>
|
|
<span class="kn">import</span> <span class="nn">time</span>
|
|
<span class="kn">import</span> <span class="nn">urlparse</span>
|
|
<span class="kn">import</span> <span class="nn">bisect</span>
|
|
<span class="kn">import</span> <span class="nn">warnings</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="kn">from</span> <span class="nn">cStringIO</span> <span class="k">import</span> <span class="n">StringIO</span>
|
|
<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
|
|
<span class="kn">from</span> <span class="nn">StringIO</span> <span class="k">import</span> <span class="n">StringIO</span>
|
|
|
|
<span class="c1"># check for SSL</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="kn">import</span> <span class="nn">ssl</span>
|
|
<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
|
|
<span class="n">_have_ssl</span> <span class="o">=</span> <span class="kc">False</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">_have_ssl</span> <span class="o">=</span> <span class="kc">True</span>
|
|
|
|
<span class="kn">from</span> <span class="nn">urllib</span> <span class="k">import</span> <span class="p">(</span><span class="n">unwrap</span><span class="p">,</span> <span class="n">unquote</span><span class="p">,</span> <span class="n">splittype</span><span class="p">,</span> <span class="n">splithost</span><span class="p">,</span> <span class="n">quote</span><span class="p">,</span>
|
|
<span class="n">addinfourl</span><span class="p">,</span> <span class="n">splitport</span><span class="p">,</span> <span class="n">splittag</span><span class="p">,</span> <span class="n">toBytes</span><span class="p">,</span>
|
|
<span class="n">splitattr</span><span class="p">,</span> <span class="n">ftpwrapper</span><span class="p">,</span> <span class="n">splituser</span><span class="p">,</span> <span class="n">splitpasswd</span><span class="p">,</span> <span class="n">splitvalue</span><span class="p">)</span>
|
|
|
|
<span class="c1"># support for FileHandler, proxies via environment variables</span>
|
|
<span class="kn">from</span> <span class="nn">urllib</span> <span class="k">import</span> <span class="n">localhost</span><span class="p">,</span> <span class="n">url2pathname</span><span class="p">,</span> <span class="n">getproxies</span><span class="p">,</span> <span class="n">proxy_bypass</span>
|
|
|
|
<span class="c1"># used in User-Agent header sent</span>
|
|
<span class="n">__version__</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">version</span><span class="p">[:</span><span class="mi">3</span><span class="p">]</span>
|
|
|
|
<span class="n">_opener</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<div class="viewcode-block" id="urlopen"><a class="viewcode-back" href="../binaryninja.downloadprovider.urlopen.html#binaryninja.downloadprovider.urlopen">[docs]</a><span class="k">def</span> <span class="nf">urlopen</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">socket</span><span class="o">.</span><span class="n">_GLOBAL_DEFAULT_TIMEOUT</span><span class="p">,</span>
|
|
<span class="n">cafile</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">capath</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cadefault</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
|
<span class="k">global</span> <span class="n">_opener</span>
|
|
<span class="k">if</span> <span class="n">cafile</span> <span class="ow">or</span> <span class="n">capath</span> <span class="ow">or</span> <span class="n">cadefault</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">context</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
|
|
<span class="s2">"You can't pass both context and any of cafile, capath, and "</span>
|
|
<span class="s2">"cadefault"</span>
|
|
<span class="p">)</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">_have_ssl</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'SSL support not available'</span><span class="p">)</span>
|
|
<span class="n">context</span> <span class="o">=</span> <span class="n">ssl</span><span class="o">.</span><span class="n">create_default_context</span><span class="p">(</span><span class="n">purpose</span><span class="o">=</span><span class="n">ssl</span><span class="o">.</span><span class="n">Purpose</span><span class="o">.</span><span class="n">SERVER_AUTH</span><span class="p">,</span>
|
|
<span class="n">cafile</span><span class="o">=</span><span class="n">cafile</span><span class="p">,</span>
|
|
<span class="n">capath</span><span class="o">=</span><span class="n">capath</span><span class="p">)</span>
|
|
<span class="n">https_handler</span> <span class="o">=</span> <span class="n">HTTPSHandler</span><span class="p">(</span><span class="n">context</span><span class="o">=</span><span class="n">context</span><span class="p">)</span>
|
|
<span class="n">opener</span> <span class="o">=</span> <span class="n">build_opener</span><span class="p">(</span><span class="n">https_handler</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="n">context</span><span class="p">:</span>
|
|
<span class="n">https_handler</span> <span class="o">=</span> <span class="n">HTTPSHandler</span><span class="p">(</span><span class="n">context</span><span class="o">=</span><span class="n">context</span><span class="p">)</span>
|
|
<span class="n">opener</span> <span class="o">=</span> <span class="n">build_opener</span><span class="p">(</span><span class="n">https_handler</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="n">_opener</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">_opener</span> <span class="o">=</span> <span class="n">opener</span> <span class="o">=</span> <span class="n">build_opener</span><span class="p">()</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">opener</span> <span class="o">=</span> <span class="n">_opener</span>
|
|
<span class="k">return</span> <span class="n">opener</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">timeout</span><span class="p">)</span></div>
|
|
|
|
<div class="viewcode-block" id="install_opener"><a class="viewcode-back" href="../binaryninja.downloadprovider.install_opener.html#binaryninja.downloadprovider.install_opener">[docs]</a><span class="k">def</span> <span class="nf">install_opener</span><span class="p">(</span><span class="n">opener</span><span class="p">):</span>
|
|
<span class="k">global</span> <span class="n">_opener</span>
|
|
<span class="n">_opener</span> <span class="o">=</span> <span class="n">opener</span></div>
|
|
|
|
<span class="c1"># do these error classes make sense?</span>
|
|
<span class="c1"># make sure all of the IOError stuff is overridden. we just want to be</span>
|
|
<span class="c1"># subtypes.</span>
|
|
|
|
<div class="viewcode-block" id="URLError"><a class="viewcode-back" href="../binaryninja.downloadprovider.URLError.html#binaryninja.downloadprovider.URLError">[docs]</a><span class="k">class</span> <span class="nc">URLError</span><span class="p">(</span><span class="ne">IOError</span><span class="p">):</span>
|
|
<span class="c1"># URLError is a sub-type of IOError, but it doesn't share any of</span>
|
|
<span class="c1"># the implementation. need to override __init__ and __str__.</span>
|
|
<span class="c1"># It sets self.args for compatibility with other EnvironmentError</span>
|
|
<span class="c1"># subclasses, but args doesn't have the typical format with errno in</span>
|
|
<span class="c1"># slot 0 and strerror in slot 1. This may be better than nothing.</span>
|
|
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">reason</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">args</span> <span class="o">=</span> <span class="n">reason</span><span class="p">,</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">reason</span> <span class="o">=</span> <span class="n">reason</span>
|
|
|
|
<span class="k">def</span> <span class="nf">__str__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="s1">'<urlopen error </span><span class="si">%s</span><span class="s1">>'</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">reason</span></div>
|
|
|
|
<span class="k">class</span> <span class="nc">HTTPError</span><span class="p">(</span><span class="n">URLError</span><span class="p">,</span> <span class="n">addinfourl</span><span class="p">):</span>
|
|
<span class="sd">"""Raised when HTTP error occurs, but also acts like non-error return"""</span>
|
|
<span class="n">__super_init</span> <span class="o">=</span> <span class="n">addinfourl</span><span class="o">.</span><span class="fm">__init__</span>
|
|
|
|
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">url</span><span class="p">,</span> <span class="n">code</span><span class="p">,</span> <span class="n">msg</span><span class="p">,</span> <span class="n">hdrs</span><span class="p">,</span> <span class="n">fp</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">code</span> <span class="o">=</span> <span class="n">code</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">msg</span> <span class="o">=</span> <span class="n">msg</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">hdrs</span> <span class="o">=</span> <span class="n">hdrs</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">fp</span> <span class="o">=</span> <span class="n">fp</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">filename</span> <span class="o">=</span> <span class="n">url</span>
|
|
<span class="c1"># The addinfourl classes depend on fp being a valid file</span>
|
|
<span class="c1"># object. In some cases, the HTTPError may not have a valid</span>
|
|
<span class="c1"># file object. If this happens, the simplest workaround is to</span>
|
|
<span class="c1"># not initialize the base classes.</span>
|
|
<span class="k">if</span> <span class="n">fp</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">__super_init</span><span class="p">(</span><span class="n">fp</span><span class="p">,</span> <span class="n">hdrs</span><span class="p">,</span> <span class="n">url</span><span class="p">,</span> <span class="n">code</span><span class="p">)</span>
|
|
|
|
<span class="k">def</span> <span class="nf">__str__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="s1">'HTTP Error </span><span class="si">%s</span><span class="s1">: </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">code</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">msg</span><span class="p">)</span>
|
|
|
|
<span class="c1"># since URLError specifies a .reason attribute, HTTPError should also</span>
|
|
<span class="c1"># provide this attribute. See issue13211 fo discussion.</span>
|
|
<span class="nd">@property</span>
|
|
<span class="k">def</span> <span class="nf">reason</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">msg</span>
|
|
|
|
<span class="k">def</span> <span class="nf">info</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hdrs</span>
|
|
|
|
<span class="c1"># copied from cookielib.py</span>
|
|
<span class="n">_cut_port_re</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">":\d+$"</span><span class="p">)</span>
|
|
<span class="k">def</span> <span class="nf">request_host</span><span class="p">(</span><span class="n">request</span><span class="p">):</span>
|
|
<span class="sd">"""Return request-host, as defined by RFC 2965.</span>
|
|
|
|
<span class="sd"> Variation from RFC: returned value is lowercased, for convenient</span>
|
|
<span class="sd"> comparison.</span>
|
|
|
|
<span class="sd"> """</span>
|
|
<span class="n">url</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">get_full_url</span><span class="p">()</span>
|
|
<span class="n">host</span> <span class="o">=</span> <span class="n">urlparse</span><span class="o">.</span><span class="n">urlparse</span><span class="p">(</span><span class="n">url</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="n">host</span> <span class="o">==</span> <span class="s2">""</span><span class="p">:</span>
|
|
<span class="n">host</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">get_header</span><span class="p">(</span><span class="s2">"Host"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
|
|
|
<span class="c1"># remove port, if present</span>
|
|
<span class="n">host</span> <span class="o">=</span> <span class="n">_cut_port_re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s2">""</span><span class="p">,</span> <span class="n">host</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">host</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
|
|
|
<span class="k">class</span> <span class="nc">Request</span><span class="p">:</span>
|
|
|
|
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="p">{},</span>
|
|
<span class="n">origin_req_host</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">unverifiable</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
|
<span class="c1"># unwrap('<URL:type://host/path>') --> 'type://host/path'</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">__original</span> <span class="o">=</span> <span class="n">unwrap</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">__original</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">__fragment</span> <span class="o">=</span> <span class="n">splittag</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__original</span><span class="p">)</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">type</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="c1"># self.__r_type is what's left after doing the splittype</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">host</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">_tunnel_host</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="n">data</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">headers</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">headers</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">add_header</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">unredirected_hdrs</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<span class="k">if</span> <span class="n">origin_req_host</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">origin_req_host</span> <span class="o">=</span> <span class="n">request_host</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">origin_req_host</span> <span class="o">=</span> <span class="n">origin_req_host</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">unverifiable</span> <span class="o">=</span> <span class="n">unverifiable</span>
|
|
|
|
<span class="k">def</span> <span class="nf">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attr</span><span class="p">):</span>
|
|
<span class="c1"># XXX this is a fallback mechanism to guard against these</span>
|
|
<span class="c1"># methods getting called in a non-standard order. this may be</span>
|
|
<span class="c1"># too complicated and/or unnecessary.</span>
|
|
<span class="c1"># XXX should the __r_XXX attributes be public?</span>
|
|
<span class="k">if</span> <span class="n">attr</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'_Request__r_type'</span><span class="p">,</span> <span class="s1">'_Request__r_host'</span><span class="p">):</span>
|
|
<span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">'get_'</span> <span class="o">+</span> <span class="n">attr</span><span class="p">[</span><span class="mi">12</span><span class="p">:])()</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">[</span><span class="n">attr</span><span class="p">]</span>
|
|
<span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">,</span> <span class="n">attr</span>
|
|
|
|
<span class="k">def</span> <span class="nf">get_method</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_data</span><span class="p">():</span>
|
|
<span class="k">return</span> <span class="s2">"POST"</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="s2">"GET"</span>
|
|
|
|
<span class="c1"># XXX these helper methods are lame</span>
|
|
|
|
<span class="k">def</span> <span class="nf">add_data</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="n">data</span>
|
|
|
|
<span class="k">def</span> <span class="nf">has_data</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
|
|
|
<span class="k">def</span> <span class="nf">get_data</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span>
|
|
|
|
<span class="k">def</span> <span class="nf">get_full_url</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">__fragment</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="s1">'</span><span class="si">%s</span><span class="s1">#</span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__original</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">__fragment</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">__original</span>
|
|
|
|
<span class="k">def</span> <span class="nf">get_type</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">type</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">type</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">__r_type</span> <span class="o">=</span> <span class="n">splittype</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__original</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">type</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">,</span> <span class="s2">"unknown url type: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">__original</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">type</span>
|
|
|
|
<span class="k">def</span> <span class="nf">get_host</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">host</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">__r_host</span> <span class="o">=</span> <span class="n">splithost</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__r_type</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">:</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">host</span> <span class="o">=</span> <span class="n">unquote</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">host</span>
|
|
|
|
<span class="k">def</span> <span class="nf">get_selector</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">__r_host</span>
|
|
|
|
<span class="k">def</span> <span class="nf">set_proxy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">host</span><span class="p">,</span> <span class="nb">type</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">type</span> <span class="o">==</span> <span class="s1">'https'</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_tunnel_host</span><span class="p">:</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">_tunnel_host</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">host</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">type</span> <span class="o">=</span> <span class="nb">type</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">__r_host</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">__original</span>
|
|
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">host</span> <span class="o">=</span> <span class="n">host</span>
|
|
|
|
<span class="k">def</span> <span class="nf">has_proxy</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">__r_host</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">__original</span>
|
|
|
|
<span class="k">def</span> <span class="nf">get_origin_req_host</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">origin_req_host</span>
|
|
|
|
<span class="k">def</span> <span class="nf">is_unverifiable</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">unverifiable</span>
|
|
|
|
<span class="k">def</span> <span class="nf">add_header</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">val</span><span class="p">):</span>
|
|
<span class="c1"># useful for something like authentication</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="n">key</span><span class="o">.</span><span class="n">capitalize</span><span class="p">()]</span> <span class="o">=</span> <span class="n">val</span>
|
|
|
|
<span class="k">def</span> <span class="nf">add_unredirected_header</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">val</span><span class="p">):</span>
|
|
<span class="c1"># will not be added to a redirected request</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">unredirected_hdrs</span><span class="p">[</span><span class="n">key</span><span class="o">.</span><span class="n">capitalize</span><span class="p">()]</span> <span class="o">=</span> <span class="n">val</span>
|
|
|
|
<span class="k">def</span> <span class="nf">has_header</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">header_name</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="p">(</span><span class="n">header_name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">headers</span> <span class="ow">or</span>
|
|
<span class="n">header_name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">unredirected_hdrs</span><span class="p">)</span>
|
|
|
|
<span class="k">def</span> <span class="nf">get_header</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">header_name</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
|
|
<span class="n">header_name</span><span class="p">,</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">unredirected_hdrs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">header_name</span><span class="p">,</span> <span class="n">default</span><span class="p">))</span>
|
|
|
|
<span class="k">def</span> <span class="nf">header_items</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="n">hdrs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">unredirected_hdrs</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="n">hdrs</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">headers</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">hdrs</span><span class="o">.</span><span class="n">items</span><span class="p">()</span>
|
|
|
|
<span class="k">class</span> <span class="nc">OpenerDirector</span><span class="p">:</span>
|
|
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="n">client_version</span> <span class="o">=</span> <span class="s2">"Python-urllib/</span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">__version__</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">addheaders</span> <span class="o">=</span> <span class="p">[(</span><span class="s1">'User-agent'</span><span class="p">,</span> <span class="n">client_version</span><span class="p">)]</span>
|
|
<span class="c1"># self.handlers is retained only for backward compatibility</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">handlers</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="c1"># manage the individual handlers</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">handle_open</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">handle_error</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">process_response</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">process_request</span> <span class="o">=</span> <span class="p">{}</span>
|
|
|
|
<span class="k">def</span> <span class="nf">add_handler</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">handler</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">handler</span><span class="p">,</span> <span class="s2">"add_parent"</span><span class="p">):</span>
|
|
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"expected BaseHandler instance, got </span><span class="si">%r</span><span class="s2">"</span> <span class="o">%</span>
|
|
<span class="nb">type</span><span class="p">(</span><span class="n">handler</span><span class="p">))</span>
|
|
|
|
<span class="n">added</span> <span class="o">=</span> <span class="kc">False</span>
|
|
<span class="k">for</span> <span class="n">meth</span> <span class="ow">in</span> <span class="nb">dir</span><span class="p">(</span><span class="n">handler</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="n">meth</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"redirect_request"</span><span class="p">,</span> <span class="s2">"do_open"</span><span class="p">,</span> <span class="s2">"proxy_open"</span><span class="p">]:</span>
|
|
<span class="c1"># oops, coincidental match</span>
|
|
<span class="k">continue</span>
|
|
|
|
<span class="n">i</span> <span class="o">=</span> <span class="n">meth</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s2">"_"</span><span class="p">)</span>
|
|
<span class="n">protocol</span> <span class="o">=</span> <span class="n">meth</span><span class="p">[:</span><span class="n">i</span><span class="p">]</span>
|
|
<span class="n">condition</span> <span class="o">=</span> <span class="n">meth</span><span class="p">[</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="p">:]</span>
|
|
|
|
<span class="k">if</span> <span class="n">condition</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"error"</span><span class="p">):</span>
|
|
<span class="n">j</span> <span class="o">=</span> <span class="n">condition</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s2">"_"</span><span class="p">)</span> <span class="o">+</span> <span class="n">i</span> <span class="o">+</span> <span class="mi">1</span>
|
|
<span class="n">kind</span> <span class="o">=</span> <span class="n">meth</span><span class="p">[</span><span class="n">j</span><span class="o">+</span><span class="mi">1</span><span class="p">:]</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">kind</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">kind</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
|
|
<span class="k">pass</span>
|
|
<span class="n">lookup</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">handle_error</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">protocol</span><span class="p">,</span> <span class="p">{})</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">handle_error</span><span class="p">[</span><span class="n">protocol</span><span class="p">]</span> <span class="o">=</span> <span class="n">lookup</span>
|
|
<span class="k">elif</span> <span class="n">condition</span> <span class="o">==</span> <span class="s2">"open"</span><span class="p">:</span>
|
|
<span class="n">kind</span> <span class="o">=</span> <span class="n">protocol</span>
|
|
<span class="n">lookup</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">handle_open</span>
|
|
<span class="k">elif</span> <span class="n">condition</span> <span class="o">==</span> <span class="s2">"response"</span><span class="p">:</span>
|
|
<span class="n">kind</span> <span class="o">=</span> <span class="n">protocol</span>
|
|
<span class="n">lookup</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">process_response</span>
|
|
<span class="k">elif</span> <span class="n">condition</span> <span class="o">==</span> <span class="s2">"request"</span><span class="p">:</span>
|
|
<span class="n">kind</span> <span class="o">=</span> <span class="n">protocol</span>
|
|
<span class="n">lookup</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">process_request</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">continue</span>
|
|
|
|
<span class="n">handlers</span> <span class="o">=</span> <span class="n">lookup</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="n">kind</span><span class="p">,</span> <span class="p">[])</span>
|
|
<span class="k">if</span> <span class="n">handlers</span><span class="p">:</span>
|
|
<span class="n">bisect</span><span class="o">.</span><span class="n">insort</span><span class="p">(</span><span class="n">handlers</span><span class="p">,</span> <span class="n">handler</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">handlers</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">handler</span><span class="p">)</span>
|
|
<span class="n">added</span> <span class="o">=</span> <span class="kc">True</span>
|
|
|
|
<span class="k">if</span> <span class="n">added</span><span class="p">:</span>
|
|
<span class="n">bisect</span><span class="o">.</span><span class="n">insort</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">handlers</span><span class="p">,</span> <span class="n">handler</span><span class="p">)</span>
|
|
<span class="n">handler</span><span class="o">.</span><span class="n">add_parent</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
|
|
|
|
<span class="k">def</span> <span class="nf">close</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="c1"># Only exists for backwards compatibility.</span>
|
|
<span class="k">pass</span>
|
|
|
|
<span class="k">def</span> <span class="nf">_call_chain</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">chain</span><span class="p">,</span> <span class="n">kind</span><span class="p">,</span> <span class="n">meth_name</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
|
|
<span class="c1"># Handlers raise an exception if no one else should try to handle</span>
|
|
<span class="c1"># the request, or return None if they can't but another handler</span>
|
|
<span class="c1"># could. Otherwise, they return the response.</span>
|
|
<span class="n">handlers</span> <span class="o">=</span> <span class="n">chain</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">kind</span><span class="p">,</span> <span class="p">())</span>
|
|
<span class="k">for</span> <span class="n">handler</span> <span class="ow">in</span> <span class="n">handlers</span><span class="p">:</span>
|
|
<span class="n">func</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">handler</span><span class="p">,</span> <span class="n">meth_name</span><span class="p">)</span>
|
|
|
|
<span class="n">result</span> <span class="o">=</span> <span class="n">func</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">result</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="n">result</span>
|
|
|
|
<span class="k">def</span> <span class="nf">open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fullurl</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">socket</span><span class="o">.</span><span class="n">_GLOBAL_DEFAULT_TIMEOUT</span><span class="p">):</span>
|
|
<span class="c1"># accept a URL or a Request object</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">fullurl</span><span class="p">,</span> <span class="n">basestring</span><span class="p">):</span>
|
|
<span class="n">req</span> <span class="o">=</span> <span class="n">Request</span><span class="p">(</span><span class="n">fullurl</span><span class="p">,</span> <span class="n">data</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">req</span> <span class="o">=</span> <span class="n">fullurl</span>
|
|
<span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">req</span><span class="o">.</span><span class="n">add_data</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
|
|
|
|
<span class="n">req</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="n">timeout</span>
|
|
<span class="n">protocol</span> <span class="o">=</span> <span class="n">req</span><span class="o">.</span><span class="n">get_type</span><span class="p">()</span>
|
|
|
|
<span class="c1"># pre-process request</span>
|
|
<span class="n">meth_name</span> <span class="o">=</span> <span class="n">protocol</span><span class="o">+</span><span class="s2">"_request"</span>
|
|
<span class="k">for</span> <span class="n">processor</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">process_request</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">protocol</span><span class="p">,</span> <span class="p">[]):</span>
|
|
<span class="n">meth</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">processor</span><span class="p">,</span> <span class="n">meth_name</span><span class="p">)</span>
|
|
<span class="n">req</span> <span class="o">=</span> <span class="n">meth</span><span class="p">(</span><span class="n">req</span><span class="p">)</span>
|
|
|
|
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_open</span><span class="p">(</span><span class="n">req</span><span class="p">,</span> <span class="n">data</span><span class="p">)</span>
|
|
|
|
<span class="c1"># post-process response</span>
|
|
<span class="n">meth_name</span> <span class="o">=</span> <span class="n">protocol</span><span class="o">+</span><span class="s2">"_response"</span>
|
|
<span class="k">for</span> <span class="n">processor</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">process_response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">protocol</span><span class="p">,</span> <span class="p">[]):</span>
|
|
<span class="n">meth</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">processor</span><span class="p">,</span> <span class="n">meth_name</span><span class="p">)</span>
|
|
<span class="n">response</span> <span class="o">=</span> <span class="n">meth</span><span class="p">(</span><span class="n">req</span><span class="p">,</span> <span class="n">response</span><span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="n">response</span>
|
|
|
|
<span class="k">def</span> <span class="nf">_open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
|
<span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_call_chain</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">handle_open</span><span class="p">,</span> <span class="s1">'default'</span><span class="p">,</span>
|
|
<span class="s1">'default_open'</span><span class="p">,</span> <span class="n">req</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">result</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="n">result</span>
|
|
|
|
<span class="n">protocol</span> <span class="o">=</span> <span class="n">req</span><span class="o">.</span><span class="n">get_type</span><span class="p">()</span>
|
|
<span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_call_chain</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">handle_open</span><span class="p">,</span> <span class="n">protocol</span><span class="p">,</span> <span class="n">protocol</span> <span class="o">+</span>
|
|
<span class="s1">'_open'</span><span class="p">,</span> <span class="n">req</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">result</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="n">result</span>
|
|
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_call_chain</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">handle_open</span><span class="p">,</span> <span class="s1">'unknown'</span><span class="p">,</span>
|
|
<span class="s1">'unknown_open'</span><span class="p">,</span> <span class="n">req</span><span class="p">)</span>
|
|
|
|
<span class="k">def</span> <span class="nf">error</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">proto</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="n">proto</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'http'</span><span class="p">,</span> <span class="s1">'https'</span><span class="p">):</span>
|
|
<span class="c1"># XXX http[s] protocols are special-cased</span>
|
|
<span class="nb">dict</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">handle_error</span><span class="p">[</span><span class="s1">'http'</span><span class="p">]</span> <span class="c1"># https is not different than http</span>
|
|
<span class="n">proto</span> <span class="o">=</span> <span class="n">args</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="c1"># YUCK!</span>
|
|
<span class="n">meth_name</span> <span class="o">=</span> <span class="s1">'http_error_</span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">proto</span>
|
|
<span class="n">http_err</span> <span class="o">=</span> <span class="mi">1</span>
|
|
<span class="n">orig_args</span> <span class="o">=</span> <span class="n">args</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="nb">dict</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">handle_error</span>
|
|
<span class="n">meth_name</span> <span class="o">=</span> <span class="n">proto</span> <span class="o">+</span> <span class="s1">'_error'</span>
|
|
<span class="n">http_err</span> <span class="o">=</span> <span class="mi">0</span>
|
|
<span class="n">args</span> <span class="o">=</span> <span class="p">(</span><span class="nb">dict</span><span class="p">,</span> <span class="n">proto</span><span class="p">,</span> <span class="n">meth_name</span><span class="p">)</span> <span class="o">+</span> <span class="n">args</span>
|
|
<span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_call_chain</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">result</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="n">result</span>
|
|
|
|
<span class="k">if</span> <span class="n">http_err</span><span class="p">:</span>
|
|
<span class="n">args</span> <span class="o">=</span> <span class="p">(</span><span class="nb">dict</span><span class="p">,</span> <span class="s1">'default'</span><span class="p">,</span> <span class="s1">'http_error_default'</span><span class="p">)</span> <span class="o">+</span> <span class="n">orig_args</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_call_chain</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">)</span>
|
|
|
|
<span class="c1"># XXX probably also want an abstract factory that knows when it makes</span>
|
|
<span class="c1"># sense to skip a superclass in favor of a subclass and when it might</span>
|
|
<span class="c1"># make sense to include both</span>
|
|
|
|
<div class="viewcode-block" id="build_opener"><a class="viewcode-back" href="../binaryninja.downloadprovider.build_opener.html#binaryninja.downloadprovider.build_opener">[docs]</a><span class="k">def</span> <span class="nf">build_opener</span><span class="p">(</span><span class="o">*</span><span class="n">handlers</span><span class="p">):</span>
|
|
<span class="sd">"""Create an opener object from a list of handlers.</span>
|
|
|
|
<span class="sd"> The opener will use several default handlers, including support</span>
|
|
<span class="sd"> for HTTP, FTP and when applicable, HTTPS.</span>
|
|
|
|
<span class="sd"> If any of the handlers passed as arguments are subclasses of the</span>
|
|
<span class="sd"> default handlers, the default handlers will not be used.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="kn">import</span> <span class="nn">types</span>
|
|
<span class="k">def</span> <span class="nf">isclass</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="p">(</span><span class="n">types</span><span class="o">.</span><span class="n">ClassType</span><span class="p">,</span> <span class="nb">type</span><span class="p">))</span>
|
|
|
|
<span class="n">opener</span> <span class="o">=</span> <span class="n">OpenerDirector</span><span class="p">()</span>
|
|
<span class="n">default_classes</span> <span class="o">=</span> <span class="p">[</span><span class="n">ProxyHandler</span><span class="p">,</span> <span class="n">UnknownHandler</span><span class="p">,</span> <span class="n">HTTPHandler</span><span class="p">,</span>
|
|
<span class="n">HTTPDefaultErrorHandler</span><span class="p">,</span> <span class="n">HTTPRedirectHandler</span><span class="p">,</span>
|
|
<span class="n">FTPHandler</span><span class="p">,</span> <span class="n">FileHandler</span><span class="p">,</span> <span class="n">HTTPErrorProcessor</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">httplib</span><span class="p">,</span> <span class="s1">'HTTPS'</span><span class="p">):</span>
|
|
<span class="n">default_classes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">HTTPSHandler</span><span class="p">)</span>
|
|
<span class="n">skip</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
|
|
<span class="k">for</span> <span class="n">klass</span> <span class="ow">in</span> <span class="n">default_classes</span><span class="p">:</span>
|
|
<span class="k">for</span> <span class="n">check</span> <span class="ow">in</span> <span class="n">handlers</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">isclass</span><span class="p">(</span><span class="n">check</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="nb">issubclass</span><span class="p">(</span><span class="n">check</span><span class="p">,</span> <span class="n">klass</span><span class="p">):</span>
|
|
<span class="n">skip</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">klass</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">check</span><span class="p">,</span> <span class="n">klass</span><span class="p">):</span>
|
|
<span class="n">skip</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">klass</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">klass</span> <span class="ow">in</span> <span class="n">skip</span><span class="p">:</span>
|
|
<span class="n">default_classes</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">klass</span><span class="p">)</span>
|
|
|
|
<span class="k">for</span> <span class="n">klass</span> <span class="ow">in</span> <span class="n">default_classes</span><span class="p">:</span>
|
|
<span class="n">opener</span><span class="o">.</span><span class="n">add_handler</span><span class="p">(</span><span class="n">klass</span><span class="p">())</span>
|
|
|
|
<span class="k">for</span> <span class="n">h</span> <span class="ow">in</span> <span class="n">handlers</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">isclass</span><span class="p">(</span><span class="n">h</span><span class="p">):</span>
|
|
<span class="n">h</span> <span class="o">=</span> <span class="n">h</span><span class="p">()</span>
|
|
<span class="n">opener</span><span class="o">.</span><span class="n">add_handler</span><span class="p">(</span><span class="n">h</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">opener</span></div>
|
|
|
|
<span class="k">class</span> <span class="nc">BaseHandler</span><span class="p">:</span>
|
|
<span class="n">handler_order</span> <span class="o">=</span> <span class="mi">500</span>
|
|
|
|
<span class="k">def</span> <span class="nf">add_parent</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">parent</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">parent</span> <span class="o">=</span> <span class="n">parent</span>
|
|
|
|
<span class="k">def</span> <span class="nf">close</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="c1"># Only exists for backwards compatibility</span>
|
|
<span class="k">pass</span>
|
|
|
|
<span class="k">def</span> <span class="nf">__lt__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="s2">"handler_order"</span><span class="p">):</span>
|
|
<span class="c1"># Try to preserve the old behavior of having custom classes</span>
|
|
<span class="c1"># inserted after default ones (works only for custom user</span>
|
|
<span class="c1"># classes which are not aware of handler_order).</span>
|
|
<span class="k">return</span> <span class="kc">True</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">handler_order</span> <span class="o"><</span> <span class="n">other</span><span class="o">.</span><span class="n">handler_order</span>
|
|
|
|
|
|
<span class="k">class</span> <span class="nc">HTTPErrorProcessor</span><span class="p">(</span><span class="n">BaseHandler</span><span class="p">):</span>
|
|
<span class="sd">"""Process HTTP error responses."""</span>
|
|
<span class="n">handler_order</span> <span class="o">=</span> <span class="mi">1000</span> <span class="c1"># after all other processing</span>
|
|
|
|
<span class="k">def</span> <span class="nf">http_response</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">,</span> <span class="n">response</span><span class="p">):</span>
|
|
<span class="n">code</span><span class="p">,</span> <span class="n">msg</span><span class="p">,</span> <span class="n">hdrs</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">code</span><span class="p">,</span> <span class="n">response</span><span class="o">.</span><span class="n">msg</span><span class="p">,</span> <span class="n">response</span><span class="o">.</span><span class="n">info</span><span class="p">()</span>
|
|
|
|
<span class="c1"># According to RFC 2616, "2xx" code indicates that the client's</span>
|
|
<span class="c1"># request was successfully received, understood, and accepted.</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="mi">200</span> <span class="o"><=</span> <span class="n">code</span> <span class="o"><</span> <span class="mi">300</span><span class="p">):</span>
|
|
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parent</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
|
|
<span class="s1">'http'</span><span class="p">,</span> <span class="n">request</span><span class="p">,</span> <span class="n">response</span><span class="p">,</span> <span class="n">code</span><span class="p">,</span> <span class="n">msg</span><span class="p">,</span> <span class="n">hdrs</span><span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="n">response</span>
|
|
|
|
<span class="n">https_response</span> <span class="o">=</span> <span class="n">http_response</span>
|
|
|
|
<span class="k">class</span> <span class="nc">HTTPDefaultErrorHandler</span><span class="p">(</span><span class="n">BaseHandler</span><span class="p">):</span>
|
|
<span class="k">def</span> <span class="nf">http_error_default</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">fp</span><span class="p">,</span> <span class="n">code</span><span class="p">,</span> <span class="n">msg</span><span class="p">,</span> <span class="n">hdrs</span><span class="p">):</span>
|
|
<span class="k">raise</span> <span class="n">HTTPError</span><span class="p">(</span><span class="n">req</span><span class="o">.</span><span class="n">get_full_url</span><span class="p">(),</span> <span class="n">code</span><span class="p">,</span> <span class="n">msg</span><span class="p">,</span> <span class="n">hdrs</span><span class="p">,</span> <span class="n">fp</span><span class="p">)</span>
|
|
|
|
<span class="k">class</span> <span class="nc">HTTPRedirectHandler</span><span class="p">(</span><span class="n">BaseHandler</span><span class="p">):</span>
|
|
<span class="c1"># maximum number of redirections to any single URL</span>
|
|
<span class="c1"># this is needed because of the state that cookies introduce</span>
|
|
<span class="n">max_repeats</span> <span class="o">=</span> <span class="mi">4</span>
|
|
<span class="c1"># maximum total number of redirections (regardless of URL) before</span>
|
|
<span class="c1"># assuming we're in a loop</span>
|
|
<span class="n">max_redirections</span> <span class="o">=</span> <span class="mi">10</span>
|
|
|
|
<span class="k">def</span> <span class="nf">redirect_request</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">fp</span><span class="p">,</span> <span class="n">code</span><span class="p">,</span> <span class="n">msg</span><span class="p">,</span> <span class="n">headers</span><span class="p">,</span> <span class="n">newurl</span><span class="p">):</span>
|
|
<span class="sd">"""Return a Request or None in response to a redirect.</span>
|
|
|
|
<span class="sd"> This is called by the http_error_30x methods when a</span>
|
|
<span class="sd"> redirection response is received. If a redirection should</span>
|
|
<span class="sd"> take place, return a new Request to allow http_error_30x to</span>
|
|
<span class="sd"> perform the redirect. Otherwise, raise HTTPError if no-one</span>
|
|
<span class="sd"> else should try to handle this url. Return None if you can't</span>
|
|
<span class="sd"> but another Handler might.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">m</span> <span class="o">=</span> <span class="n">req</span><span class="o">.</span><span class="n">get_method</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="p">(</span><span class="n">code</span> <span class="ow">in</span> <span class="p">(</span><span class="mi">301</span><span class="p">,</span> <span class="mi">302</span><span class="p">,</span> <span class="mi">303</span><span class="p">,</span> <span class="mi">307</span><span class="p">)</span> <span class="ow">and</span> <span class="n">m</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"GET"</span><span class="p">,</span> <span class="s2">"HEAD"</span><span class="p">)</span>
|
|
<span class="ow">or</span> <span class="n">code</span> <span class="ow">in</span> <span class="p">(</span><span class="mi">301</span><span class="p">,</span> <span class="mi">302</span><span class="p">,</span> <span class="mi">303</span><span class="p">)</span> <span class="ow">and</span> <span class="n">m</span> <span class="o">==</span> <span class="s2">"POST"</span><span class="p">):</span>
|
|
<span class="c1"># Strictly (according to RFC 2616), 301 or 302 in response</span>
|
|
<span class="c1"># to a POST MUST NOT cause a redirection without confirmation</span>
|
|
<span class="c1"># from the user (of urllib2, in this case). In practice,</span>
|
|
<span class="c1"># essentially all clients do redirect in this case, so we</span>
|
|
<span class="c1"># do the same.</span>
|
|
<span class="c1"># be conciliant with URIs containing a space</span>
|
|
<span class="n">newurl</span> <span class="o">=</span> <span class="n">newurl</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">' '</span><span class="p">,</span> <span class="s1">'%20'</span><span class="p">)</span>
|
|
<span class="n">newheaders</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">((</span><span class="n">k</span><span class="p">,</span><span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span><span class="n">v</span> <span class="ow">in</span> <span class="n">req</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">items</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="n">k</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"content-length"</span><span class="p">,</span> <span class="s2">"content-type"</span><span class="p">)</span>
|
|
<span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">Request</span><span class="p">(</span><span class="n">newurl</span><span class="p">,</span>
|
|
<span class="n">headers</span><span class="o">=</span><span class="n">newheaders</span><span class="p">,</span>
|
|
<span class="n">origin_req_host</span><span class="o">=</span><span class="n">req</span><span class="o">.</span><span class="n">get_origin_req_host</span><span class="p">(),</span>
|
|
<span class="n">unverifiable</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">HTTPError</span><span class="p">(</span><span class="n">req</span><span class="o">.</span><span class="n">get_full_url</span><span class="p">(),</span> <span class="n">code</span><span class="p">,</span> <span class="n">msg</span><span class="p">,</span> <span class="n">headers</span><span class="p">,</span> <span class="n">fp</span><span class="p">)</span>
|
|
|
|
<span class="c1"># Implementation note: To avoid the server sending us into an</span>
|
|
<span class="c1"># infinite loop, the request object needs to track what URLs we</span>
|
|
<span class="c1"># have already seen. Do this by adding a handler-specific</span>
|
|
<span class="c1"># attribute to the Request object.</span>
|
|
<span class="k">def</span> <span class="nf">http_error_302</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">fp</span><span class="p">,</span> <span class="n">code</span><span class="p">,</span> <span class="n">msg</span><span class="p">,</span> <span class="n">headers</span><span class="p">):</span>
|
|
<span class="c1"># Some servers (incorrectly) return multiple Location headers</span>
|
|
<span class="c1"># (so probably same goes for URI). Use first header.</span>
|
|
<span class="k">if</span> <span class="s1">'location'</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
|
|
<span class="n">newurl</span> <span class="o">=</span> <span class="n">headers</span><span class="o">.</span><span class="n">getheaders</span><span class="p">(</span><span class="s1">'location'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
|
<span class="k">elif</span> <span class="s1">'uri'</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
|
|
<span class="n">newurl</span> <span class="o">=</span> <span class="n">headers</span><span class="o">.</span><span class="n">getheaders</span><span class="p">(</span><span class="s1">'uri'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">return</span>
|
|
|
|
<span class="c1"># fix a possible malformed URL</span>
|
|
<span class="n">urlparts</span> <span class="o">=</span> <span class="n">urlparse</span><span class="o">.</span><span class="n">urlparse</span><span class="p">(</span><span class="n">newurl</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">urlparts</span><span class="o">.</span><span class="n">path</span> <span class="ow">and</span> <span class="n">urlparts</span><span class="o">.</span><span class="n">netloc</span><span class="p">:</span>
|
|
<span class="n">urlparts</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">urlparts</span><span class="p">)</span>
|
|
<span class="n">urlparts</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"/"</span>
|
|
<span class="n">newurl</span> <span class="o">=</span> <span class="n">urlparse</span><span class="o">.</span><span class="n">urlunparse</span><span class="p">(</span><span class="n">urlparts</span><span class="p">)</span>
|
|
|
|
<span class="n">newurl</span> <span class="o">=</span> <span class="n">urlparse</span><span class="o">.</span><span class="n">urljoin</span><span class="p">(</span><span class="n">req</span><span class="o">.</span><span class="n">get_full_url</span><span class="p">(),</span> <span class="n">newurl</span><span class="p">)</span>
|
|
|
|
<span class="c1"># For security reasons we do not allow redirects to protocols</span>
|
|
<span class="c1"># other than HTTP, HTTPS or FTP.</span>
|
|
<span class="n">newurl_lower</span> <span class="o">=</span> <span class="n">newurl</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="n">newurl_lower</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'http://'</span><span class="p">)</span> <span class="ow">or</span>
|
|
<span class="n">newurl_lower</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'https://'</span><span class="p">)</span> <span class="ow">or</span>
|
|
<span class="n">newurl_lower</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'ftp://'</span><span class="p">)):</span>
|
|
<span class="k">raise</span> <span class="n">HTTPError</span><span class="p">(</span><span class="n">newurl</span><span class="p">,</span> <span class="n">code</span><span class="p">,</span>
|
|
<span class="n">msg</span> <span class="o">+</span> <span class="s2">" - Redirection to url '</span><span class="si">%s</span><span class="s2">' is not allowed"</span> <span class="o">%</span>
|
|
<span class="n">newurl</span><span class="p">,</span>
|
|
<span class="n">headers</span><span class="p">,</span> <span class="n">fp</span><span class="p">)</span>
|
|
|
|
<span class="c1"># XXX Probably want to forget about the state of the current</span>
|
|
<span class="c1"># request, although that might interact poorly with other</span>
|
|
<span class="c1"># handlers that also use handler-specific request attributes</span>
|
|
<span class="n">new</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">redirect_request</span><span class="p">(</span><span class="n">req</span><span class="p">,</span> <span class="n">fp</span><span class="p">,</span> <span class="n">code</span><span class="p">,</span> <span class="n">msg</span><span class="p">,</span> <span class="n">headers</span><span class="p">,</span> <span class="n">newurl</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">new</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">return</span>
|
|
|
|
<span class="c1"># loop detection</span>
|
|
<span class="c1"># .redirect_dict has a key url if url was previously visited.</span>
|
|
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">req</span><span class="p">,</span> <span class="s1">'redirect_dict'</span><span class="p">):</span>
|
|
<span class="n">visited</span> <span class="o">=</span> <span class="n">new</span><span class="o">.</span><span class="n">redirect_dict</span> <span class="o">=</span> <span class="n">req</span><span class="o">.</span><span class="n">redirect_dict</span>
|
|
<span class="k">if</span> <span class="p">(</span><span class="n">visited</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">newurl</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_repeats</span> <span class="ow">or</span>
|
|
<span class="nb">len</span><span class="p">(</span><span class="n">visited</span><span class="p">)</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_redirections</span><span class="p">):</span>
|
|
<span class="k">raise</span> <span class="n">HTTPError</span><span class="p">(</span><span class="n">req</span><span class="o">.</span><span class="n">get_full_url</span><span class="p">(),</span> <span class="n">code</span><span class="p">,</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">inf_msg</span> <span class="o">+</span> <span class="n">msg</span><span class="p">,</span> <span class="n">headers</span><span class="p">,</span> <span class="n">fp</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">visited</span> <span class="o">=</span> <span class="n">new</span><span class="o">.</span><span class="n">redirect_dict</span> <span class="o">=</span> <span class="n">req</span><span class="o">.</span><span class="n">redirect_dict</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<span class="n">visited</span><span class="p">[</span><span class="n">newurl</span><span class="p">]</span> <span class="o">=</span> <span class="n">visited</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">newurl</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span>
|
|
|
|
<span class="c1"># Don't close the fp until we are sure that we won't use it</span>
|
|
<span class="c1"># with HTTPError.</span>
|
|
<span class="n">fp</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
|
|
<span class="n">fp</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
|
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">parent</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">new</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">req</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
|
|
|
|
<span class="n">http_error_301</span> <span class="o">=</span> <span class="n">http_error_303</span> <span class="o">=</span> <span class="n">http_error_307</span> <span class="o">=</span> <span class="n">http_error_302</span>
|
|
|
|
<span class="n">inf_msg</span> <span class="o">=</span> <span class="s2">"The HTTP server returned a redirect error that would "</span> \
|
|
<span class="s2">"lead to an infinite loop.</span><span class="se">\n</span><span class="s2">"</span> \
|
|
<span class="s2">"The last 30x error message was:</span><span class="se">\n</span><span class="s2">"</span>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">_parse_proxy</span><span class="p">(</span><span class="n">proxy</span><span class="p">):</span>
|
|
<span class="sd">"""Return (scheme, user, password, host/port) given a URL or an authority.</span>
|
|
|
|
<span class="sd"> If a URL is supplied, it must have an authority (host:port) component.</span>
|
|
<span class="sd"> According to RFC 3986, having an authority component means the URL must</span>
|
|
<span class="sd"> have two slashes after the scheme:</span>
|
|
|
|
<span class="sd"> >>> _parse_proxy('file:/ftp.example.com/')</span>
|
|
<span class="sd"> Traceback (most recent call last):</span>
|
|
<span class="sd"> ValueError: proxy URL with no authority: 'file:/ftp.example.com/'</span>
|
|
|
|
<span class="sd"> The first three items of the returned tuple may be None.</span>
|
|
|
|
<span class="sd"> Examples of authority parsing:</span>
|
|
|
|
<span class="sd"> >>> _parse_proxy('proxy.example.com')</span>
|
|
<span class="sd"> (None, None, None, 'proxy.example.com')</span>
|
|
<span class="sd"> >>> _parse_proxy('proxy.example.com:3128')</span>
|
|
<span class="sd"> (None, None, None, 'proxy.example.com:3128')</span>
|
|
|
|
<span class="sd"> The authority component may optionally include userinfo (assumed to be</span>
|
|
<span class="sd"> username:password):</span>
|
|
|
|
<span class="sd"> >>> _parse_proxy('joe:password@proxy.example.com')</span>
|
|
<span class="sd"> (None, 'joe', 'password', 'proxy.example.com')</span>
|
|
<span class="sd"> >>> _parse_proxy('joe:password@proxy.example.com:3128')</span>
|
|
<span class="sd"> (None, 'joe', 'password', 'proxy.example.com:3128')</span>
|
|
|
|
<span class="sd"> Same examples, but with URLs instead:</span>
|
|
|
|
<span class="sd"> >>> _parse_proxy('http://proxy.example.com/')</span>
|
|
<span class="sd"> ('http', None, None, 'proxy.example.com')</span>
|
|
<span class="sd"> >>> _parse_proxy('http://proxy.example.com:3128/')</span>
|
|
<span class="sd"> ('http', None, None, 'proxy.example.com:3128')</span>
|
|
<span class="sd"> >>> _parse_proxy('http://joe:password@proxy.example.com/')</span>
|
|
<span class="sd"> ('http', 'joe', 'password', 'proxy.example.com')</span>
|
|
<span class="sd"> >>> _parse_proxy('http://joe:password@proxy.example.com:3128')</span>
|
|
<span class="sd"> ('http', 'joe', 'password', 'proxy.example.com:3128')</span>
|
|
|
|
<span class="sd"> Everything after the authority is ignored:</span>
|
|
|
|
<span class="sd"> >>> _parse_proxy('ftp://joe:password@proxy.example.com/rubbish:3128')</span>
|
|
<span class="sd"> ('ftp', 'joe', 'password', 'proxy.example.com')</span>
|
|
|
|
<span class="sd"> Test for no trailing '/' case:</span>
|
|
|
|
<span class="sd"> >>> _parse_proxy('http://joe:password@proxy.example.com')</span>
|
|
<span class="sd"> ('http', 'joe', 'password', 'proxy.example.com')</span>
|
|
|
|
<span class="sd"> """</span>
|
|
<span class="n">scheme</span><span class="p">,</span> <span class="n">r_scheme</span> <span class="o">=</span> <span class="n">splittype</span><span class="p">(</span><span class="n">proxy</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">r_scheme</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"/"</span><span class="p">):</span>
|
|
<span class="c1"># authority</span>
|
|
<span class="n">scheme</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">authority</span> <span class="o">=</span> <span class="n">proxy</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="c1"># URL</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">r_scheme</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"//"</span><span class="p">):</span>
|
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"proxy URL with no authority: </span><span class="si">%r</span><span class="s2">"</span> <span class="o">%</span> <span class="n">proxy</span><span class="p">)</span>
|
|
<span class="c1"># We have an authority, so for RFC 3986-compliant URLs (by ss 3.</span>
|
|
<span class="c1"># and 3.3.), path is empty or starts with '/'</span>
|
|
<span class="n">end</span> <span class="o">=</span> <span class="n">r_scheme</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s2">"/"</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">end</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
|
|
<span class="n">end</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">authority</span> <span class="o">=</span> <span class="n">r_scheme</span><span class="p">[</span><span class="mi">2</span><span class="p">:</span><span class="n">end</span><span class="p">]</span>
|
|
<span class="n">userinfo</span><span class="p">,</span> <span class="n">hostport</span> <span class="o">=</span> <span class="n">splituser</span><span class="p">(</span><span class="n">authority</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">userinfo</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">user</span><span class="p">,</span> <span class="n">password</span> <span class="o">=</span> <span class="n">splitpasswd</span><span class="p">(</span><span class="n">userinfo</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">user</span> <span class="o">=</span> <span class="n">password</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">return</span> <span class="n">scheme</span><span class="p">,</span> <span class="n">user</span><span class="p">,</span> <span class="n">password</span><span class="p">,</span> <span class="n">hostport</span>
|
|
|
|
<div class="viewcode-block" id="ProxyHandler"><a class="viewcode-back" href="../binaryninja.downloadprovider.ProxyHandler.html#binaryninja.downloadprovider.ProxyHandler">[docs]</a><span class="k">class</span> <span class="nc">ProxyHandler</span><span class="p">(</span><span class="n">BaseHandler</span><span class="p">):</span>
|
|
<span class="c1"># Proxies must be in front</span>
|
|
<span class="n">handler_order</span> <span class="o">=</span> <span class="mi">100</span>
|
|
|
|
<div class="viewcode-block" id="ProxyHandler.__init__"><a class="viewcode-back" href="../binaryninja.downloadprovider.ProxyHandler.html#binaryninja.downloadprovider.ProxyHandler.__init__">[docs]</a> <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">proxies</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="n">proxies</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">proxies</span> <span class="o">=</span> <span class="n">getproxies</span><span class="p">()</span>
|
|
<span class="k">assert</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">proxies</span><span class="p">,</span> <span class="s1">'has_key'</span><span class="p">),</span> <span class="s2">"proxies must be a mapping"</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">proxies</span> <span class="o">=</span> <span class="n">proxies</span>
|
|
<span class="k">for</span> <span class="nb">type</span><span class="p">,</span> <span class="n">url</span> <span class="ow">in</span> <span class="n">proxies</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
|
<span class="nb">setattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">'</span><span class="si">%s</span><span class="s1">_open'</span> <span class="o">%</span> <span class="nb">type</span><span class="p">,</span>
|
|
<span class="k">lambda</span> <span class="n">r</span><span class="p">,</span> <span class="n">proxy</span><span class="o">=</span><span class="n">url</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="nb">type</span><span class="p">,</span> <span class="n">meth</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">proxy_open</span><span class="p">:</span> \
|
|
<span class="n">meth</span><span class="p">(</span><span class="n">r</span><span class="p">,</span> <span class="n">proxy</span><span class="p">,</span> <span class="nb">type</span><span class="p">))</span></div>
|
|
|
|
<span class="k">def</span> <span class="nf">proxy_open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">proxy</span><span class="p">,</span> <span class="nb">type</span><span class="p">):</span>
|
|
<span class="n">orig_type</span> <span class="o">=</span> <span class="n">req</span><span class="o">.</span><span class="n">get_type</span><span class="p">()</span>
|
|
<span class="n">proxy_type</span><span class="p">,</span> <span class="n">user</span><span class="p">,</span> <span class="n">password</span><span class="p">,</span> <span class="n">hostport</span> <span class="o">=</span> <span class="n">_parse_proxy</span><span class="p">(</span><span class="n">proxy</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">proxy_type</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">proxy_type</span> <span class="o">=</span> <span class="n">orig_type</span>
|
|
|
|
<span class="k">if</span> <span class="n">req</span><span class="o">.</span><span class="n">host</span> <span class="ow">and</span> <span class="n">proxy_bypass</span><span class="p">(</span><span class="n">req</span><span class="o">.</span><span class="n">host</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="kc">None</span>
|
|
|
|
<span class="k">if</span> <span class="n">user</span> <span class="ow">and</span> <span class="n">password</span><span class="p">:</span>
|
|
<span class="n">user_pass</span> <span class="o">=</span> <span class="s1">'</span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="p">(</span><span class="n">unquote</span><span class="p">(</span><span class="n">user</span><span class="p">),</span> <span class="n">unquote</span><span class="p">(</span><span class="n">password</span><span class="p">))</span>
|
|
<span class="n">creds</span> <span class="o">=</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64encode</span><span class="p">(</span><span class="n">user_pass</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
|
<span class="n">req</span><span class="o">.</span><span class="n">add_header</span><span class="p">(</span><span class="s1">'Proxy-authorization'</span><span class="p">,</span> <span class="s1">'Basic '</span> <span class="o">+</span> <span class="n">creds</span><span class="p">)</span>
|
|
<span class="n">hostport</span> <span class="o">=</span> <span class="n">unquote</span><span class="p">(</span><span class="n">hostport</span><span class="p">)</span>
|
|
<span class="n">req</span><span class="o">.</span><span class="n">set_proxy</span><span class="p">(</span><span class="n">hostport</span><span class="p">,</span> <span class="n">proxy_type</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">orig_type</span> <span class="o">==</span> <span class="n">proxy_type</span> <span class="ow">or</span> <span class="n">orig_type</span> <span class="o">==</span> <span class="s1">'https'</span><span class="p">:</span>
|
|
<span class="c1"># let other handlers take care of it</span>
|
|
<span class="k">return</span> <span class="kc">None</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="c1"># need to start over, because the other handlers don't</span>
|
|
<span class="c1"># grok the proxy's URL type</span>
|
|
<span class="c1"># e.g. if we have a constructor arg proxies like so:</span>
|
|
<span class="c1"># {'http': 'ftp://proxy.example.com'}, we may end up turning</span>
|
|
<span class="c1"># a request for http://acme.example.com/a into one for</span>
|
|
<span class="c1"># ftp://proxy.example.com/a</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">parent</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">req</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">req</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span></div>
|
|
|
|
<span class="k">class</span> <span class="nc">HTTPPasswordMgr</span><span class="p">:</span>
|
|
|
|
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">passwd</span> <span class="o">=</span> <span class="p">{}</span>
|
|
|
|
<span class="k">def</span> <span class="nf">add_password</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">realm</span><span class="p">,</span> <span class="n">uri</span><span class="p">,</span> <span class="n">user</span><span class="p">,</span> <span class="n">passwd</span><span class="p">):</span>
|
|
<span class="c1"># uri could be a single URI or a sequence</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">uri</span><span class="p">,</span> <span class="n">basestring</span><span class="p">):</span>
|
|
<span class="n">uri</span> <span class="o">=</span> <span class="p">[</span><span class="n">uri</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">realm</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">passwd</span><span class="p">:</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">passwd</span><span class="p">[</span><span class="n">realm</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<span class="k">for</span> <span class="n">default_port</span> <span class="ow">in</span> <span class="kc">True</span><span class="p">,</span> <span class="kc">False</span><span class="p">:</span>
|
|
<span class="n">reduced_uri</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span>
|
|
<span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">reduce_uri</span><span class="p">(</span><span class="n">u</span><span class="p">,</span> <span class="n">default_port</span><span class="p">)</span> <span class="k">for</span> <span class="n">u</span> <span class="ow">in</span> <span class="n">uri</span><span class="p">])</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">passwd</span><span class="p">[</span><span class="n">realm</span><span class="p">][</span><span class="n">reduced_uri</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">user</span><span class="p">,</span> <span class="n">passwd</span><span class="p">)</span>
|
|
|
|
<span class="k">def</span> <span class="nf">find_user_password</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">realm</span><span class="p">,</span> <span class="n">authuri</span><span class="p">):</span>
|
|
<span class="n">domains</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">passwd</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">realm</span><span class="p">,</span> <span class="p">{})</span>
|
|
<span class="k">for</span> <span class="n">default_port</span> <span class="ow">in</span> <span class="kc">True</span><span class="p">,</span> <span class="kc">False</span><span class="p">:</span>
|
|
<span class="n">reduced_authuri</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">reduce_uri</span><span class="p">(</span><span class="n">authuri</span><span class="p">,</span> <span class="n">default_port</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">uris</span><span class="p">,</span> <span class="n">authinfo</span> <span class="ow">in</span> <span class="n">domains</span><span class="o">.</span><span class="n">iteritems</span><span class="p">():</span>
|
|
<span class="k">for</span> <span class="n">uri</span> <span class="ow">in</span> <span class="n">uris</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_suburi</span><span class="p">(</span><span class="n">uri</span><span class="p">,</span> <span class="n">reduced_authuri</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="n">authinfo</span>
|
|
<span class="k">return</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">None</span>
|
|
|
|
<span class="k">def</span> <span class="nf">reduce_uri</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">uri</span><span class="p">,</span> <span class="n">default_port</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
|
<span class="sd">"""Accept authority or URI and extract only the authority and path."""</span>
|
|
<span class="c1"># note HTTP URLs do not have a userinfo component</span>
|
|
<span class="n">parts</span> <span class="o">=</span> <span class="n">urlparse</span><span class="o">.</span><span class="n">urlsplit</span><span class="p">(</span><span class="n">uri</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">parts</span><span class="p">[</span><span class="mi">1</span><span class="p">]:</span>
|
|
<span class="c1"># URI</span>
|
|
<span class="n">scheme</span> <span class="o">=</span> <span class="n">parts</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
|
<span class="n">authority</span> <span class="o">=</span> <span class="n">parts</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
|
|
<span class="n">path</span> <span class="o">=</span> <span class="n">parts</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="ow">or</span> <span class="s1">'/'</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="c1"># host or host:port</span>
|
|
<span class="n">scheme</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">authority</span> <span class="o">=</span> <span class="n">uri</span>
|
|
<span class="n">path</span> <span class="o">=</span> <span class="s1">'/'</span>
|
|
<span class="n">host</span><span class="p">,</span> <span class="n">port</span> <span class="o">=</span> <span class="n">splitport</span><span class="p">(</span><span class="n">authority</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">default_port</span> <span class="ow">and</span> <span class="n">port</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">scheme</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">dport</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"http"</span><span class="p">:</span> <span class="mi">80</span><span class="p">,</span>
|
|
<span class="s2">"https"</span><span class="p">:</span> <span class="mi">443</span><span class="p">,</span>
|
|
<span class="p">}</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">scheme</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">dport</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">authority</span> <span class="o">=</span> <span class="s2">"</span><span class="si">%s</span><span class="s2">:</span><span class="si">%d</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">dport</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">authority</span><span class="p">,</span> <span class="n">path</span>
|
|
|
|
<span class="k">def</span> <span class="nf">is_suburi</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">base</span><span class="p">,</span> <span class="n">test</span><span class="p">):</span>
|
|
<span class="sd">"""Check if test is below base in a URI tree</span>
|
|
|
|
<span class="sd"> Both args must be URIs in reduced form.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">if</span> <span class="n">base</span> <span class="o">==</span> <span class="n">test</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="kc">True</span>
|
|
<span class="k">if</span> <span class="n">base</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="n">test</span><span class="p">[</span><span class="mi">0</span><span class="p">]:</span>
|
|
<span class="k">return</span> <span class="kc">False</span>
|
|
<span class="n">common</span> <span class="o">=</span> <span class="n">posixpath</span><span class="o">.</span><span class="n">commonprefix</span><span class="p">((</span><span class="n">base</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">test</span><span class="p">[</span><span class="mi">1</span><span class="p">]))</span>
|
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">common</span><span class="p">)</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">base</span><span class="p">[</span><span class="mi">1</span><span class="p">]):</span>
|
|
<span class="k">return</span> <span class="kc">True</span>
|
|
<span class="k">return</span> <span class="kc">False</span>
|
|
|
|
|
|
<span class="k">class</span> <span class="nc">HTTPPasswordMgrWithDefaultRealm</span><span class="p">(</span><span class="n">HTTPPasswordMgr</span><span class="p">):</span>
|
|
|
|
<span class="k">def</span> <span class="nf">find_user_password</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">realm</span><span class="p">,</span> <span class="n">authuri</span><span class="p">):</span>
|
|
<span class="n">user</span><span class="p">,</span> <span class="n">password</span> <span class="o">=</span> <span class="n">HTTPPasswordMgr</span><span class="o">.</span><span class="n">find_user_password</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">realm</span><span class="p">,</span>
|
|
<span class="n">authuri</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">user</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="n">user</span><span class="p">,</span> <span class="n">password</span>
|
|
<span class="k">return</span> <span class="n">HTTPPasswordMgr</span><span class="o">.</span><span class="n">find_user_password</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="n">authuri</span><span class="p">)</span>
|
|
|
|
|
|
<span class="k">class</span> <span class="nc">AbstractBasicAuthHandler</span><span class="p">:</span>
|
|
|
|
<span class="c1"># XXX this allows for multiple auth-schemes, but will stupidly pick</span>
|
|
<span class="c1"># the last one with a realm specified.</span>
|
|
|
|
<span class="c1"># allow for double- and single-quoted realm values</span>
|
|
<span class="c1"># (single quotes are a violation of the RFC, but appear in the wild)</span>
|
|
<span class="n">rx</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="s1">'(?:.*,)*[ </span><span class="se">\t</span><span class="s1">]*([^ </span><span class="se">\t</span><span class="s1">]+)[ </span><span class="se">\t</span><span class="s1">]+'</span>
|
|
<span class="s1">'realm=(["</span><span class="se">\'</span><span class="s1">]?)([^"</span><span class="se">\'</span><span class="s1">]*)</span><span class="se">\\</span><span class="s1">2'</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">I</span><span class="p">)</span>
|
|
|
|
<span class="c1"># XXX could pre-emptively send auth info already accepted (RFC 2617,</span>
|
|
<span class="c1"># end of section 2, and section 1.2 immediately after "credentials"</span>
|
|
<span class="c1"># production).</span>
|
|
|
|
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">password_mgr</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="n">password_mgr</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">password_mgr</span> <span class="o">=</span> <span class="n">HTTPPasswordMgr</span><span class="p">()</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">passwd</span> <span class="o">=</span> <span class="n">password_mgr</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">add_password</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">passwd</span><span class="o">.</span><span class="n">add_password</span>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">http_error_auth_reqed</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">authreq</span><span class="p">,</span> <span class="n">host</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">headers</span><span class="p">):</span>
|
|
<span class="c1"># host may be an authority (without userinfo) or a URL with an</span>
|
|
<span class="c1"># authority</span>
|
|
<span class="c1"># XXX could be multiple headers</span>
|
|
<span class="n">authreq</span> <span class="o">=</span> <span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">authreq</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">authreq</span><span class="p">:</span>
|
|
<span class="n">mo</span> <span class="o">=</span> <span class="n">AbstractBasicAuthHandler</span><span class="o">.</span><span class="n">rx</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">authreq</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">mo</span><span class="p">:</span>
|
|
<span class="n">scheme</span><span class="p">,</span> <span class="n">quote</span><span class="p">,</span> <span class="n">realm</span> <span class="o">=</span> <span class="n">mo</span><span class="o">.</span><span class="n">groups</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="n">quote</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'"'</span><span class="p">,</span> <span class="s2">"'"</span><span class="p">]:</span>
|
|
<span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">"Basic Auth Realm was unquoted"</span><span class="p">,</span>
|
|
<span class="ne">UserWarning</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">scheme</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'basic'</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">retry_http_basic_auth</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">realm</span><span class="p">)</span>
|
|
|
|
<span class="k">def</span> <span class="nf">retry_http_basic_auth</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">host</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">realm</span><span class="p">):</span>
|
|
<span class="n">user</span><span class="p">,</span> <span class="n">pw</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">passwd</span><span class="o">.</span><span class="n">find_user_password</span><span class="p">(</span><span class="n">realm</span><span class="p">,</span> <span class="n">host</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">pw</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">raw</span> <span class="o">=</span> <span class="s2">"</span><span class="si">%s</span><span class="s2">:</span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">user</span><span class="p">,</span> <span class="n">pw</span><span class="p">)</span>
|
|
<span class="n">auth</span> <span class="o">=</span> <span class="s1">'Basic </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64encode</span><span class="p">(</span><span class="n">raw</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="n">req</span><span class="o">.</span><span class="n">get_header</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">auth_header</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="o">==</span> <span class="n">auth</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="kc">None</span>
|
|
<span class="n">req</span><span class="o">.</span><span class="n">add_unredirected_header</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">auth_header</span><span class="p">,</span> <span class="n">auth</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">parent</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">req</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">req</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="kc">None</span>
|
|
|
|
|
|
<span class="k">class</span> <span class="nc">HTTPBasicAuthHandler</span><span class="p">(</span><span class="n">AbstractBasicAuthHandler</span><span class="p">,</span> <span class="n">BaseHandler</span><span class="p">):</span>
|
|
|
|
<span class="n">auth_header</span> <span class="o">=</span> <span class="s1">'Authorization'</span>
|
|
|
|
<span class="k">def</span> <span class="nf">http_error_401</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">fp</span><span class="p">,</span> <span class="n">code</span><span class="p">,</span> <span class="n">msg</span><span class="p">,</span> <span class="n">headers</span><span class="p">):</span>
|
|
<span class="n">url</span> <span class="o">=</span> <span class="n">req</span><span class="o">.</span><span class="n">get_full_url</span><span class="p">()</span>
|
|
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">http_error_auth_reqed</span><span class="p">(</span><span class="s1">'www-authenticate'</span><span class="p">,</span>
|
|
<span class="n">url</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">headers</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">response</span>
|
|
|
|
|
|
<span class="k">class</span> <span class="nc">ProxyBasicAuthHandler</span><span class="p">(</span><span class="n">AbstractBasicAuthHandler</span><span class="p">,</span> <span class="n">BaseHandler</span><span class="p">):</span>
|
|
|
|
<span class="n">auth_header</span> <span class="o">=</span> <span class="s1">'Proxy-authorization'</span>
|
|
|
|
<span class="k">def</span> <span class="nf">http_error_407</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">fp</span><span class="p">,</span> <span class="n">code</span><span class="p">,</span> <span class="n">msg</span><span class="p">,</span> <span class="n">headers</span><span class="p">):</span>
|
|
<span class="c1"># http_error_auth_reqed requires that there is no userinfo component in</span>
|
|
<span class="c1"># authority. Assume there isn't one, since urllib2 does not (and</span>
|
|
<span class="c1"># should not, RFC 3986 s. 3.2.1) support requests for URLs containing</span>
|
|
<span class="c1"># userinfo.</span>
|
|
<span class="n">authority</span> <span class="o">=</span> <span class="n">req</span><span class="o">.</span><span class="n">get_host</span><span class="p">()</span>
|
|
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">http_error_auth_reqed</span><span class="p">(</span><span class="s1">'proxy-authenticate'</span><span class="p">,</span>
|
|
<span class="n">authority</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">headers</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">response</span>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">randombytes</span><span class="p">(</span><span class="n">n</span><span class="p">):</span>
|
|
<span class="sd">"""Return n random bytes."""</span>
|
|
<span class="c1"># Use /dev/urandom if it is available. Fall back to random module</span>
|
|
<span class="c1"># if not. It might be worthwhile to extend this function to use</span>
|
|
<span class="c1"># other platform-specific mechanisms for getting random bytes.</span>
|
|
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="s2">"/dev/urandom"</span><span class="p">):</span>
|
|
<span class="n">f</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="s2">"/dev/urandom"</span><span class="p">)</span>
|
|
<span class="n">s</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="n">n</span><span class="p">)</span>
|
|
<span class="n">f</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
|
<span class="k">return</span> <span class="n">s</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">L</span> <span class="o">=</span> <span class="p">[</span><span class="nb">chr</span><span class="p">(</span><span class="n">random</span><span class="o">.</span><span class="n">randrange</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">256</span><span class="p">))</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n</span><span class="p">)]</span>
|
|
<span class="k">return</span> <span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">L</span><span class="p">)</span>
|
|
|
|
<span class="k">class</span> <span class="nc">AbstractDigestAuthHandler</span><span class="p">:</span>
|
|
<span class="c1"># Digest authentication is specified in RFC 2617.</span>
|
|
|
|
<span class="c1"># XXX The client does not inspect the Authentication-Info header</span>
|
|
<span class="c1"># in a successful response.</span>
|
|
|
|
<span class="c1"># XXX It should be possible to test this implementation against</span>
|
|
<span class="c1"># a mock server that just generates a static set of challenges.</span>
|
|
|
|
<span class="c1"># XXX qop="auth-int" supports is shaky</span>
|
|
|
|
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">passwd</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="n">passwd</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">passwd</span> <span class="o">=</span> <span class="n">HTTPPasswordMgr</span><span class="p">()</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">passwd</span> <span class="o">=</span> <span class="n">passwd</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">add_password</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">passwd</span><span class="o">.</span><span class="n">add_password</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">retried</span> <span class="o">=</span> <span class="mi">0</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">nonce_count</span> <span class="o">=</span> <span class="mi">0</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">last_nonce</span> <span class="o">=</span> <span class="kc">None</span>
|
|
|
|
<span class="k">def</span> <span class="nf">reset_retry_count</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">retried</span> <span class="o">=</span> <span class="mi">0</span>
|
|
|
|
<span class="k">def</span> <span class="nf">http_error_auth_reqed</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">auth_header</span><span class="p">,</span> <span class="n">host</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">headers</span><span class="p">):</span>
|
|
<span class="n">authreq</span> <span class="o">=</span> <span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">auth_header</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">retried</span> <span class="o">></span> <span class="mi">5</span><span class="p">:</span>
|
|
<span class="c1"># Don't fail endlessly - if we failed once, we'll probably</span>
|
|
<span class="c1"># fail a second time. Hm. Unless the Password Manager is</span>
|
|
<span class="c1"># prompting for the information. Crap. This isn't great</span>
|
|
<span class="c1"># but it's better than the current 'repeat until recursion</span>
|
|
<span class="c1"># depth exceeded' approach <wink></span>
|
|
<span class="k">raise</span> <span class="n">HTTPError</span><span class="p">(</span><span class="n">req</span><span class="o">.</span><span class="n">get_full_url</span><span class="p">(),</span> <span class="mi">401</span><span class="p">,</span> <span class="s2">"digest auth failed"</span><span class="p">,</span>
|
|
<span class="n">headers</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">retried</span> <span class="o">+=</span> <span class="mi">1</span>
|
|
<span class="k">if</span> <span class="n">authreq</span><span class="p">:</span>
|
|
<span class="n">scheme</span> <span class="o">=</span> <span class="n">authreq</span><span class="o">.</span><span class="n">split</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="n">scheme</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'digest'</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">retry_http_digest_auth</span><span class="p">(</span><span class="n">req</span><span class="p">,</span> <span class="n">authreq</span><span class="p">)</span>
|
|
|
|
<span class="k">def</span> <span class="nf">retry_http_digest_auth</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">auth</span><span class="p">):</span>
|
|
<span class="n">token</span><span class="p">,</span> <span class="n">challenge</span> <span class="o">=</span> <span class="n">auth</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">' '</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
|
<span class="n">chal</span> <span class="o">=</span> <span class="n">parse_keqv_list</span><span class="p">(</span><span class="n">parse_http_list</span><span class="p">(</span><span class="n">challenge</span><span class="p">))</span>
|
|
<span class="n">auth</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_authorization</span><span class="p">(</span><span class="n">req</span><span class="p">,</span> <span class="n">chal</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">auth</span><span class="p">:</span>
|
|
<span class="n">auth_val</span> <span class="o">=</span> <span class="s1">'Digest </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">auth</span>
|
|
<span class="k">if</span> <span class="n">req</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">auth_header</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="o">==</span> <span class="n">auth_val</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="kc">None</span>
|
|
<span class="n">req</span><span class="o">.</span><span class="n">add_unredirected_header</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">auth_header</span><span class="p">,</span> <span class="n">auth_val</span><span class="p">)</span>
|
|
<span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parent</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">req</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">req</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">resp</span>
|
|
|
|
<span class="k">def</span> <span class="nf">get_cnonce</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">nonce</span><span class="p">):</span>
|
|
<span class="c1"># The cnonce-value is an opaque</span>
|
|
<span class="c1"># quoted string value provided by the client and used by both client</span>
|
|
<span class="c1"># and server to avoid chosen plaintext attacks, to provide mutual</span>
|
|
<span class="c1"># authentication, and to provide some message integrity protection.</span>
|
|
<span class="c1"># This isn't a fabulous effort, but it's probably Good Enough.</span>
|
|
<span class="n">dig</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">sha1</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2">:</span><span class="si">%s</span><span class="s2">:</span><span class="si">%s</span><span class="s2">:</span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">nonce_count</span><span class="p">,</span> <span class="n">nonce</span><span class="p">,</span> <span class="n">time</span><span class="o">.</span><span class="n">ctime</span><span class="p">(),</span>
|
|
<span class="n">randombytes</span><span class="p">(</span><span class="mi">8</span><span class="p">)))</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
|
|
<span class="k">return</span> <span class="n">dig</span><span class="p">[:</span><span class="mi">16</span><span class="p">]</span>
|
|
|
|
<span class="k">def</span> <span class="nf">get_authorization</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">chal</span><span class="p">):</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">realm</span> <span class="o">=</span> <span class="n">chal</span><span class="p">[</span><span class="s1">'realm'</span><span class="p">]</span>
|
|
<span class="n">nonce</span> <span class="o">=</span> <span class="n">chal</span><span class="p">[</span><span class="s1">'nonce'</span><span class="p">]</span>
|
|
<span class="n">qop</span> <span class="o">=</span> <span class="n">chal</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'qop'</span><span class="p">)</span>
|
|
<span class="n">algorithm</span> <span class="o">=</span> <span class="n">chal</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'algorithm'</span><span class="p">,</span> <span class="s1">'MD5'</span><span class="p">)</span>
|
|
<span class="c1"># mod_digest doesn't send an opaque, even though it isn't</span>
|
|
<span class="c1"># supposed to be optional</span>
|
|
<span class="n">opaque</span> <span class="o">=</span> <span class="n">chal</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'opaque'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="kc">None</span>
|
|
|
|
<span class="n">H</span><span class="p">,</span> <span class="n">KD</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_algorithm_impls</span><span class="p">(</span><span class="n">algorithm</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">H</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="kc">None</span>
|
|
|
|
<span class="n">user</span><span class="p">,</span> <span class="n">pw</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">passwd</span><span class="o">.</span><span class="n">find_user_password</span><span class="p">(</span><span class="n">realm</span><span class="p">,</span> <span class="n">req</span><span class="o">.</span><span class="n">get_full_url</span><span class="p">())</span>
|
|
<span class="k">if</span> <span class="n">user</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="kc">None</span>
|
|
|
|
<span class="c1"># XXX not implemented yet</span>
|
|
<span class="k">if</span> <span class="n">req</span><span class="o">.</span><span class="n">has_data</span><span class="p">():</span>
|
|
<span class="n">entdig</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_entity_digest</span><span class="p">(</span><span class="n">req</span><span class="o">.</span><span class="n">get_data</span><span class="p">(),</span> <span class="n">chal</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">entdig</span> <span class="o">=</span> <span class="kc">None</span>
|
|
|
|
<span class="n">A1</span> <span class="o">=</span> <span class="s2">"</span><span class="si">%s</span><span class="s2">:</span><span class="si">%s</span><span class="s2">:</span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">user</span><span class="p">,</span> <span class="n">realm</span><span class="p">,</span> <span class="n">pw</span><span class="p">)</span>
|
|
<span class="n">A2</span> <span class="o">=</span> <span class="s2">"</span><span class="si">%s</span><span class="s2">:</span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">req</span><span class="o">.</span><span class="n">get_method</span><span class="p">(),</span>
|
|
<span class="c1"># XXX selector: what about proxies and full urls</span>
|
|
<span class="n">req</span><span class="o">.</span><span class="n">get_selector</span><span class="p">())</span>
|
|
<span class="k">if</span> <span class="n">qop</span> <span class="o">==</span> <span class="s1">'auth'</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">nonce</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_nonce</span><span class="p">:</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">nonce_count</span> <span class="o">+=</span> <span class="mi">1</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">nonce_count</span> <span class="o">=</span> <span class="mi">1</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">last_nonce</span> <span class="o">=</span> <span class="n">nonce</span>
|
|
|
|
<span class="n">ncvalue</span> <span class="o">=</span> <span class="s1">'</span><span class="si">%08x</span><span class="s1">'</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">nonce_count</span>
|
|
<span class="n">cnonce</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_cnonce</span><span class="p">(</span><span class="n">nonce</span><span class="p">)</span>
|
|
<span class="n">noncebit</span> <span class="o">=</span> <span class="s2">"</span><span class="si">%s</span><span class="s2">:</span><span class="si">%s</span><span class="s2">:</span><span class="si">%s</span><span class="s2">:</span><span class="si">%s</span><span class="s2">:</span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">nonce</span><span class="p">,</span> <span class="n">ncvalue</span><span class="p">,</span> <span class="n">cnonce</span><span class="p">,</span> <span class="n">qop</span><span class="p">,</span> <span class="n">H</span><span class="p">(</span><span class="n">A2</span><span class="p">))</span>
|
|
<span class="n">respdig</span> <span class="o">=</span> <span class="n">KD</span><span class="p">(</span><span class="n">H</span><span class="p">(</span><span class="n">A1</span><span class="p">),</span> <span class="n">noncebit</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="n">qop</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">respdig</span> <span class="o">=</span> <span class="n">KD</span><span class="p">(</span><span class="n">H</span><span class="p">(</span><span class="n">A1</span><span class="p">),</span> <span class="s2">"</span><span class="si">%s</span><span class="s2">:</span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">nonce</span><span class="p">,</span> <span class="n">H</span><span class="p">(</span><span class="n">A2</span><span class="p">)))</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="c1"># XXX handle auth-int.</span>
|
|
<span class="k">raise</span> <span class="n">URLError</span><span class="p">(</span><span class="s2">"qop '</span><span class="si">%s</span><span class="s2">' is not supported."</span> <span class="o">%</span> <span class="n">qop</span><span class="p">)</span>
|
|
|
|
<span class="c1"># XXX should the partial digests be encoded too?</span>
|
|
|
|
<span class="n">base</span> <span class="o">=</span> <span class="s1">'username="</span><span class="si">%s</span><span class="s1">", realm="</span><span class="si">%s</span><span class="s1">", nonce="</span><span class="si">%s</span><span class="s1">", uri="</span><span class="si">%s</span><span class="s1">", '</span> \
|
|
<span class="s1">'response="</span><span class="si">%s</span><span class="s1">"'</span> <span class="o">%</span> <span class="p">(</span><span class="n">user</span><span class="p">,</span> <span class="n">realm</span><span class="p">,</span> <span class="n">nonce</span><span class="p">,</span> <span class="n">req</span><span class="o">.</span><span class="n">get_selector</span><span class="p">(),</span>
|
|
<span class="n">respdig</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">opaque</span><span class="p">:</span>
|
|
<span class="n">base</span> <span class="o">+=</span> <span class="s1">', opaque="</span><span class="si">%s</span><span class="s1">"'</span> <span class="o">%</span> <span class="n">opaque</span>
|
|
<span class="k">if</span> <span class="n">entdig</span><span class="p">:</span>
|
|
<span class="n">base</span> <span class="o">+=</span> <span class="s1">', digest="</span><span class="si">%s</span><span class="s1">"'</span> <span class="o">%</span> <span class="n">entdig</span>
|
|
<span class="n">base</span> <span class="o">+=</span> <span class="s1">', algorithm="</span><span class="si">%s</span><span class="s1">"'</span> <span class="o">%</span> <span class="n">algorithm</span>
|
|
<span class="k">if</span> <span class="n">qop</span><span class="p">:</span>
|
|
<span class="n">base</span> <span class="o">+=</span> <span class="s1">', qop=auth, nc=</span><span class="si">%s</span><span class="s1">, cnonce="</span><span class="si">%s</span><span class="s1">"'</span> <span class="o">%</span> <span class="p">(</span><span class="n">ncvalue</span><span class="p">,</span> <span class="n">cnonce</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">base</span>
|
|
|
|
<span class="k">def</span> <span class="nf">get_algorithm_impls</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">algorithm</span><span class="p">):</span>
|
|
<span class="c1"># algorithm should be case-insensitive according to RFC2617</span>
|
|
<span class="n">algorithm</span> <span class="o">=</span> <span class="n">algorithm</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
|
|
<span class="c1"># lambdas assume digest modules are imported at the top level</span>
|
|
<span class="k">if</span> <span class="n">algorithm</span> <span class="o">==</span> <span class="s1">'MD5'</span><span class="p">:</span>
|
|
<span class="n">H</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">md5</span><span class="p">(</span><span class="n">x</span><span class="p">)</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
|
|
<span class="k">elif</span> <span class="n">algorithm</span> <span class="o">==</span> <span class="s1">'SHA'</span><span class="p">:</span>
|
|
<span class="n">H</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">sha1</span><span class="p">(</span><span class="n">x</span><span class="p">)</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
|
|
<span class="c1"># XXX MD5-sess</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Unsupported digest authentication "</span>
|
|
<span class="s2">"algorithm </span><span class="si">%r</span><span class="s2">"</span> <span class="o">%</span> <span class="n">algorithm</span><span class="o">.</span><span class="n">lower</span><span class="p">())</span>
|
|
<span class="n">KD</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">s</span><span class="p">,</span> <span class="n">d</span><span class="p">:</span> <span class="n">H</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2">:</span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="n">d</span><span class="p">))</span>
|
|
<span class="k">return</span> <span class="n">H</span><span class="p">,</span> <span class="n">KD</span>
|
|
|
|
<span class="k">def</span> <span class="nf">get_entity_digest</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">chal</span><span class="p">):</span>
|
|
<span class="c1"># XXX not implemented yet</span>
|
|
<span class="k">return</span> <span class="kc">None</span>
|
|
|
|
|
|
<span class="k">class</span> <span class="nc">HTTPDigestAuthHandler</span><span class="p">(</span><span class="n">BaseHandler</span><span class="p">,</span> <span class="n">AbstractDigestAuthHandler</span><span class="p">):</span>
|
|
<span class="sd">"""An authentication protocol defined by RFC 2069</span>
|
|
|
|
<span class="sd"> Digest authentication improves on basic authentication because it</span>
|
|
<span class="sd"> does not transmit passwords in the clear.</span>
|
|
<span class="sd"> """</span>
|
|
|
|
<span class="n">auth_header</span> <span class="o">=</span> <span class="s1">'Authorization'</span>
|
|
<span class="n">handler_order</span> <span class="o">=</span> <span class="mi">490</span> <span class="c1"># before Basic auth</span>
|
|
|
|
<span class="k">def</span> <span class="nf">http_error_401</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">fp</span><span class="p">,</span> <span class="n">code</span><span class="p">,</span> <span class="n">msg</span><span class="p">,</span> <span class="n">headers</span><span class="p">):</span>
|
|
<span class="n">host</span> <span class="o">=</span> <span class="n">urlparse</span><span class="o">.</span><span class="n">urlparse</span><span class="p">(</span><span class="n">req</span><span class="o">.</span><span class="n">get_full_url</span><span class="p">())[</span><span class="mi">1</span><span class="p">]</span>
|
|
<span class="n">retry</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">http_error_auth_reqed</span><span class="p">(</span><span class="s1">'www-authenticate'</span><span class="p">,</span>
|
|
<span class="n">host</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">headers</span><span class="p">)</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">reset_retry_count</span><span class="p">()</span>
|
|
<span class="k">return</span> <span class="n">retry</span>
|
|
|
|
|
|
<span class="k">class</span> <span class="nc">ProxyDigestAuthHandler</span><span class="p">(</span><span class="n">BaseHandler</span><span class="p">,</span> <span class="n">AbstractDigestAuthHandler</span><span class="p">):</span>
|
|
|
|
<span class="n">auth_header</span> <span class="o">=</span> <span class="s1">'Proxy-Authorization'</span>
|
|
<span class="n">handler_order</span> <span class="o">=</span> <span class="mi">490</span> <span class="c1"># before Basic auth</span>
|
|
|
|
<span class="k">def</span> <span class="nf">http_error_407</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">fp</span><span class="p">,</span> <span class="n">code</span><span class="p">,</span> <span class="n">msg</span><span class="p">,</span> <span class="n">headers</span><span class="p">):</span>
|
|
<span class="n">host</span> <span class="o">=</span> <span class="n">req</span><span class="o">.</span><span class="n">get_host</span><span class="p">()</span>
|
|
<span class="n">retry</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">http_error_auth_reqed</span><span class="p">(</span><span class="s1">'proxy-authenticate'</span><span class="p">,</span>
|
|
<span class="n">host</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="n">headers</span><span class="p">)</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">reset_retry_count</span><span class="p">()</span>
|
|
<span class="k">return</span> <span class="n">retry</span>
|
|
|
|
<span class="k">class</span> <span class="nc">AbstractHTTPHandler</span><span class="p">(</span><span class="n">BaseHandler</span><span class="p">):</span>
|
|
|
|
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">debuglevel</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">_debuglevel</span> <span class="o">=</span> <span class="n">debuglevel</span>
|
|
|
|
<span class="k">def</span> <span class="nf">set_http_debuglevel</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">level</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">_debuglevel</span> <span class="o">=</span> <span class="n">level</span>
|
|
|
|
<span class="k">def</span> <span class="nf">do_request_</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">):</span>
|
|
<span class="n">host</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">get_host</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">host</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">URLError</span><span class="p">(</span><span class="s1">'no host given'</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">request</span><span class="o">.</span><span class="n">has_data</span><span class="p">():</span> <span class="c1"># POST</span>
|
|
<span class="n">data</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">get_data</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">request</span><span class="o">.</span><span class="n">has_header</span><span class="p">(</span><span class="s1">'Content-type'</span><span class="p">):</span>
|
|
<span class="n">request</span><span class="o">.</span><span class="n">add_unredirected_header</span><span class="p">(</span>
|
|
<span class="s1">'Content-type'</span><span class="p">,</span>
|
|
<span class="s1">'application/x-www-form-urlencoded'</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">request</span><span class="o">.</span><span class="n">has_header</span><span class="p">(</span><span class="s1">'Content-length'</span><span class="p">):</span>
|
|
<span class="n">request</span><span class="o">.</span><span class="n">add_unredirected_header</span><span class="p">(</span>
|
|
<span class="s1">'Content-length'</span><span class="p">,</span> <span class="s1">'</span><span class="si">%d</span><span class="s1">'</span> <span class="o">%</span> <span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
|
|
|
|
<span class="n">sel_host</span> <span class="o">=</span> <span class="n">host</span>
|
|
<span class="k">if</span> <span class="n">request</span><span class="o">.</span><span class="n">has_proxy</span><span class="p">():</span>
|
|
<span class="n">scheme</span><span class="p">,</span> <span class="n">sel</span> <span class="o">=</span> <span class="n">splittype</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">get_selector</span><span class="p">())</span>
|
|
<span class="n">sel_host</span><span class="p">,</span> <span class="n">sel_path</span> <span class="o">=</span> <span class="n">splithost</span><span class="p">(</span><span class="n">sel</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">request</span><span class="o">.</span><span class="n">has_header</span><span class="p">(</span><span class="s1">'Host'</span><span class="p">):</span>
|
|
<span class="n">request</span><span class="o">.</span><span class="n">add_unredirected_header</span><span class="p">(</span><span class="s1">'Host'</span><span class="p">,</span> <span class="n">sel_host</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">parent</span><span class="o">.</span><span class="n">addheaders</span><span class="p">:</span>
|
|
<span class="n">name</span> <span class="o">=</span> <span class="n">name</span><span class="o">.</span><span class="n">capitalize</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">request</span><span class="o">.</span><span class="n">has_header</span><span class="p">(</span><span class="n">name</span><span class="p">):</span>
|
|
<span class="n">request</span><span class="o">.</span><span class="n">add_unredirected_header</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="n">request</span>
|
|
|
|
<span class="k">def</span> <span class="nf">do_open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">http_class</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span> <span class="o">**</span><span class="n">http_conn_args</span><span class="p">):</span>
|
|
<span class="sd">"""Return an addinfourl object for the request, using http_class.</span>
|
|
|
|
<span class="sd"> http_class must implement the HTTPConnection API from httplib.</span>
|
|
<span class="sd"> The addinfourl return value is a file-like object. It also</span>
|
|
<span class="sd"> has methods and attributes including:</span>
|
|
<span class="sd"> - info(): return a mimetools.Message object for the headers</span>
|
|
<span class="sd"> - geturl(): return the original request URL</span>
|
|
<span class="sd"> - code: HTTP status code</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">host</span> <span class="o">=</span> <span class="n">req</span><span class="o">.</span><span class="n">get_host</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">host</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">URLError</span><span class="p">(</span><span class="s1">'no host given'</span><span class="p">)</span>
|
|
|
|
<span class="c1"># will parse host:port</span>
|
|
<span class="n">h</span> <span class="o">=</span> <span class="n">http_class</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">req</span><span class="o">.</span><span class="n">timeout</span><span class="p">,</span> <span class="o">**</span><span class="n">http_conn_args</span><span class="p">)</span>
|
|
<span class="n">h</span><span class="o">.</span><span class="n">set_debuglevel</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_debuglevel</span><span class="p">)</span>
|
|
|
|
<span class="n">headers</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">req</span><span class="o">.</span><span class="n">unredirected_hdrs</span><span class="p">)</span>
|
|
<span class="n">headers</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="nb">dict</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">req</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">items</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="n">k</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">))</span>
|
|
|
|
<span class="c1"># We want to make an HTTP/1.1 request, but the addinfourl</span>
|
|
<span class="c1"># class isn't prepared to deal with a persistent connection.</span>
|
|
<span class="c1"># It will try to read all remaining data from the socket,</span>
|
|
<span class="c1"># which will block while the server waits for the next request.</span>
|
|
<span class="c1"># So make sure the connection gets closed after the (only)</span>
|
|
<span class="c1"># request.</span>
|
|
<span class="n">headers</span><span class="p">[</span><span class="s2">"Connection"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"close"</span>
|
|
<span class="n">headers</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
|
<span class="p">(</span><span class="n">name</span><span class="o">.</span><span class="n">title</span><span class="p">(),</span> <span class="n">val</span><span class="p">)</span> <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">val</span> <span class="ow">in</span> <span class="n">headers</span><span class="o">.</span><span class="n">items</span><span class="p">())</span>
|
|
|
|
<span class="k">if</span> <span class="n">req</span><span class="o">.</span><span class="n">_tunnel_host</span><span class="p">:</span>
|
|
<span class="n">tunnel_headers</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<span class="n">proxy_auth_hdr</span> <span class="o">=</span> <span class="s2">"Proxy-Authorization"</span>
|
|
<span class="k">if</span> <span class="n">proxy_auth_hdr</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
|
|
<span class="n">tunnel_headers</span><span class="p">[</span><span class="n">proxy_auth_hdr</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="n">proxy_auth_hdr</span><span class="p">]</span>
|
|
<span class="c1"># Proxy-Authorization should not be sent to origin</span>
|
|
<span class="c1"># server.</span>
|
|
<span class="k">del</span> <span class="n">headers</span><span class="p">[</span><span class="n">proxy_auth_hdr</span><span class="p">]</span>
|
|
<span class="n">h</span><span class="o">.</span><span class="n">set_tunnel</span><span class="p">(</span><span class="n">req</span><span class="o">.</span><span class="n">_tunnel_host</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">tunnel_headers</span><span class="p">)</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">h</span><span class="o">.</span><span class="n">request</span><span class="p">(</span><span class="n">req</span><span class="o">.</span><span class="n">get_method</span><span class="p">(),</span> <span class="n">req</span><span class="o">.</span><span class="n">get_selector</span><span class="p">(),</span> <span class="n">req</span><span class="o">.</span><span class="n">data</span><span class="p">,</span> <span class="n">headers</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">socket</span><span class="o">.</span><span class="n">error</span><span class="p">,</span> <span class="n">err</span><span class="p">:</span> <span class="c1"># XXX what error?</span>
|
|
<span class="n">h</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
|
<span class="k">raise</span> <span class="n">URLError</span><span class="p">(</span><span class="n">err</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">r</span> <span class="o">=</span> <span class="n">h</span><span class="o">.</span><span class="n">getresponse</span><span class="p">(</span><span class="n">buffering</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">TypeError</span><span class="p">:</span> <span class="c1"># buffering kw not supported</span>
|
|
<span class="n">r</span> <span class="o">=</span> <span class="n">h</span><span class="o">.</span><span class="n">getresponse</span><span class="p">()</span>
|
|
|
|
<span class="c1"># Pick apart the HTTPResponse object to get the addinfourl</span>
|
|
<span class="c1"># object initialized properly.</span>
|
|
|
|
<span class="c1"># Wrap the HTTPResponse object in socket's file object adapter</span>
|
|
<span class="c1"># for Windows. That adapter calls recv(), so delegate recv()</span>
|
|
<span class="c1"># to read(). This weird wrapping allows the returned object to</span>
|
|
<span class="c1"># have readline() and readlines() methods.</span>
|
|
|
|
<span class="c1"># XXX It might be better to extract the read buffering code</span>
|
|
<span class="c1"># out of socket._fileobject() and into a base class.</span>
|
|
|
|
<span class="n">r</span><span class="o">.</span><span class="n">recv</span> <span class="o">=</span> <span class="n">r</span><span class="o">.</span><span class="n">read</span>
|
|
<span class="n">fp</span> <span class="o">=</span> <span class="n">socket</span><span class="o">.</span><span class="n">_fileobject</span><span class="p">(</span><span class="n">r</span><span class="p">,</span> <span class="n">close</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
|
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">addinfourl</span><span class="p">(</span><span class="n">fp</span><span class="p">,</span> <span class="n">r</span><span class="o">.</span><span class="n">msg</span><span class="p">,</span> <span class="n">req</span><span class="o">.</span><span class="n">get_full_url</span><span class="p">())</span>
|
|
<span class="n">resp</span><span class="o">.</span><span class="n">code</span> <span class="o">=</span> <span class="n">r</span><span class="o">.</span><span class="n">status</span>
|
|
<span class="n">resp</span><span class="o">.</span><span class="n">msg</span> <span class="o">=</span> <span class="n">r</span><span class="o">.</span><span class="n">reason</span>
|
|
<span class="k">return</span> <span class="n">resp</span>
|
|
|
|
|
|
<span class="k">class</span> <span class="nc">HTTPHandler</span><span class="p">(</span><span class="n">AbstractHTTPHandler</span><span class="p">):</span>
|
|
|
|
<span class="k">def</span> <span class="nf">http_open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">req</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">do_open</span><span class="p">(</span><span class="n">httplib</span><span class="o">.</span><span class="n">HTTPConnection</span><span class="p">,</span> <span class="n">req</span><span class="p">)</span>
|
|
|
|
<span class="n">http_request</span> <span class="o">=</span> <span class="n">AbstractHTTPHandler</span><span class="o">.</span><span class="n">do_request_</span>
|
|
|
|
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">httplib</span><span class="p">,</span> <span class="s1">'HTTPS'</span><span class="p">):</span>
|
|
<span class="k">class</span> <span class="nc">HTTPSHandler</span><span class="p">(</span><span class="n">AbstractHTTPHandler</span><span class="p">):</span>
|
|
|
|
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">debuglevel</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
|
<span class="n">AbstractHTTPHandler</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">debuglevel</span><span class="p">)</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">_context</span> <span class="o">=</span> <span class="n">context</span>
|
|
|
|
<span class="k">def</span> <span class="nf">https_open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">req</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">do_open</span><span class="p">(</span><span class="n">httplib</span><span class="o">.</span><span class="n">HTTPSConnection</span><span class="p">,</span> <span class="n">req</span><span class="p">,</span>
|
|
<span class="n">context</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_context</span><span class="p">)</span>
|
|
|
|
<span class="n">https_request</span> <span class="o">=</span> <span class="n">AbstractHTTPHandler</span><span class="o">.</span><span class="n">do_request_</span>
|
|
|
|
<span class="k">class</span> <span class="nc">HTTPCookieProcessor</span><span class="p">(</span><span class="n">BaseHandler</span><span class="p">):</span>
|
|
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cookiejar</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
|
<span class="kn">import</span> <span class="nn">cookielib</span>
|
|
<span class="k">if</span> <span class="n">cookiejar</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">cookiejar</span> <span class="o">=</span> <span class="n">cookielib</span><span class="o">.</span><span class="n">CookieJar</span><span class="p">()</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">cookiejar</span> <span class="o">=</span> <span class="n">cookiejar</span>
|
|
|
|
<span class="k">def</span> <span class="nf">http_request</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">cookiejar</span><span class="o">.</span><span class="n">add_cookie_header</span><span class="p">(</span><span class="n">request</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">request</span>
|
|
|
|
<span class="k">def</span> <span class="nf">http_response</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">,</span> <span class="n">response</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">cookiejar</span><span class="o">.</span><span class="n">extract_cookies</span><span class="p">(</span><span class="n">response</span><span class="p">,</span> <span class="n">request</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">response</span>
|
|
|
|
<span class="n">https_request</span> <span class="o">=</span> <span class="n">http_request</span>
|
|
<span class="n">https_response</span> <span class="o">=</span> <span class="n">http_response</span>
|
|
|
|
<span class="k">class</span> <span class="nc">UnknownHandler</span><span class="p">(</span><span class="n">BaseHandler</span><span class="p">):</span>
|
|
<span class="k">def</span> <span class="nf">unknown_open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">req</span><span class="p">):</span>
|
|
<span class="nb">type</span> <span class="o">=</span> <span class="n">req</span><span class="o">.</span><span class="n">get_type</span><span class="p">()</span>
|
|
<span class="k">raise</span> <span class="n">URLError</span><span class="p">(</span><span class="s1">'unknown url type: </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="nb">type</span><span class="p">)</span>
|
|
|
|
<span class="k">def</span> <span class="nf">parse_keqv_list</span><span class="p">(</span><span class="n">l</span><span class="p">):</span>
|
|
<span class="sd">"""Parse list of key=value strings where keys are not duplicated."""</span>
|
|
<span class="n">parsed</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<span class="k">for</span> <span class="n">elt</span> <span class="ow">in</span> <span class="n">l</span><span class="p">:</span>
|
|
<span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="o">=</span> <span class="n">elt</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'='</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">v</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'"'</span> <span class="ow">and</span> <span class="n">v</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'"'</span><span class="p">:</span>
|
|
<span class="n">v</span> <span class="o">=</span> <span class="n">v</span><span class="p">[</span><span class="mi">1</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
|
<span class="n">parsed</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">v</span>
|
|
<span class="k">return</span> <span class="n">parsed</span>
|
|
|
|
<span class="k">def</span> <span class="nf">parse_http_list</span><span class="p">(</span><span class="n">s</span><span class="p">):</span>
|
|
<span class="sd">"""Parse lists as described by RFC 2068 Section 2.</span>
|
|
|
|
<span class="sd"> In particular, parse comma-separated lists where the elements of</span>
|
|
<span class="sd"> the list may include quoted-strings. A quoted-string could</span>
|
|
<span class="sd"> contain a comma. A non-quoted string could have quotes in the</span>
|
|
<span class="sd"> middle. Neither commas nor quotes count if they are escaped.</span>
|
|
<span class="sd"> Only double-quotes count, not single-quotes.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">res</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">part</span> <span class="o">=</span> <span class="s1">''</span>
|
|
|
|
<span class="n">escape</span> <span class="o">=</span> <span class="n">quote</span> <span class="o">=</span> <span class="kc">False</span>
|
|
<span class="k">for</span> <span class="n">cur</span> <span class="ow">in</span> <span class="n">s</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">escape</span><span class="p">:</span>
|
|
<span class="n">part</span> <span class="o">+=</span> <span class="n">cur</span>
|
|
<span class="n">escape</span> <span class="o">=</span> <span class="kc">False</span>
|
|
<span class="k">continue</span>
|
|
<span class="k">if</span> <span class="n">quote</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">cur</span> <span class="o">==</span> <span class="s1">'</span><span class="se">\\</span><span class="s1">'</span><span class="p">:</span>
|
|
<span class="n">escape</span> <span class="o">=</span> <span class="kc">True</span>
|
|
<span class="k">continue</span>
|
|
<span class="k">elif</span> <span class="n">cur</span> <span class="o">==</span> <span class="s1">'"'</span><span class="p">:</span>
|
|
<span class="n">quote</span> <span class="o">=</span> <span class="kc">False</span>
|
|
<span class="n">part</span> <span class="o">+=</span> <span class="n">cur</span>
|
|
<span class="k">continue</span>
|
|
|
|
<span class="k">if</span> <span class="n">cur</span> <span class="o">==</span> <span class="s1">','</span><span class="p">:</span>
|
|
<span class="n">res</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">part</span><span class="p">)</span>
|
|
<span class="n">part</span> <span class="o">=</span> <span class="s1">''</span>
|
|
<span class="k">continue</span>
|
|
|
|
<span class="k">if</span> <span class="n">cur</span> <span class="o">==</span> <span class="s1">'"'</span><span class="p">:</span>
|
|
<span class="n">quote</span> <span class="o">=</span> <span class="kc">True</span>
|
|
|
|
<span class="n">part</span> <span class="o">+=</span> <span class="n">cur</span>
|
|
|
|
<span class="c1"># append last part</span>
|
|
<span class="k">if</span> <span class="n">part</span><span class="p">:</span>
|
|
<span class="n">res</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">part</span><span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="p">[</span><span class="n">part</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">res</span><span class="p">]</span>
|
|
|
|
<span class="k">def</span> <span class="nf">_safe_gethostbyname</span><span class="p">(</span><span class="n">host</span><span class="p">):</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="n">socket</span><span class="o">.</span><span class="n">gethostbyname</span><span class="p">(</span><span class="n">host</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">socket</span><span class="o">.</span><span class="n">gaierror</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="kc">None</span>
|
|
|
|
<span class="k">class</span> <span class="nc">FileHandler</span><span class="p">(</span><span class="n">BaseHandler</span><span class="p">):</span>
|
|
<span class="c1"># Use local file or FTP depending on form of URL</span>
|
|
<span class="k">def</span> <span class="nf">file_open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">req</span><span class="p">):</span>
|
|
<span class="n">url</span> <span class="o">=</span> <span class="n">req</span><span class="o">.</span><span class="n">get_selector</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="n">url</span><span class="p">[:</span><span class="mi">2</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'//'</span> <span class="ow">and</span> <span class="n">url</span><span class="p">[</span><span class="mi">2</span><span class="p">:</span><span class="mi">3</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">'/'</span> <span class="ow">and</span> <span class="p">(</span><span class="n">req</span><span class="o">.</span><span class="n">host</span> <span class="ow">and</span>
|
|
<span class="n">req</span><span class="o">.</span><span class="n">host</span> <span class="o">!=</span> <span class="s1">'localhost'</span><span class="p">):</span>
|
|
<span class="n">req</span><span class="o">.</span><span class="n">type</span> <span class="o">=</span> <span class="s1">'ftp'</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">parent</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">req</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">open_local_file</span><span class="p">(</span><span class="n">req</span><span class="p">)</span>
|
|
|
|
<span class="c1"># names for the localhost</span>
|
|
<span class="n">names</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">def</span> <span class="nf">get_names</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="n">FileHandler</span><span class="o">.</span><span class="n">names</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">FileHandler</span><span class="o">.</span><span class="n">names</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span>
|
|
<span class="n">socket</span><span class="o">.</span><span class="n">gethostbyname_ex</span><span class="p">(</span><span class="s1">'localhost'</span><span class="p">)[</span><span class="mi">2</span><span class="p">]</span> <span class="o">+</span>
|
|
<span class="n">socket</span><span class="o">.</span><span class="n">gethostbyname_ex</span><span class="p">(</span><span class="n">socket</span><span class="o">.</span><span class="n">gethostname</span><span class="p">())[</span><span class="mi">2</span><span class="p">])</span>
|
|
<span class="k">except</span> <span class="n">socket</span><span class="o">.</span><span class="n">gaierror</span><span class="p">:</span>
|
|
<span class="n">FileHandler</span><span class="o">.</span><span class="n">names</span> <span class="o">=</span> <span class="p">(</span><span class="n">socket</span><span class="o">.</span><span class="n">gethostbyname</span><span class="p">(</span><span class="s1">'localhost'</span><span class="p">),)</span>
|
|
<span class="k">return</span> <span class="n">FileHandler</span><span class="o">.</span><span class="n">names</span>
|
|
|
|
<span class="c1"># not entirely sure what the rules are here</span>
|
|
<span class="k">def</span> <span class="nf">open_local_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">req</span><span class="p">):</span>
|
|
<span class="kn">import</span> <span class="nn">email.utils</span>
|
|
<span class="kn">import</span> <span class="nn">mimetypes</span>
|
|
<span class="n">host</span> <span class="o">=</span> <span class="n">req</span><span class="o">.</span><span class="n">get_host</span><span class="p">()</span>
|
|
<span class="n">filename</span> <span class="o">=</span> <span class="n">req</span><span class="o">.</span><span class="n">get_selector</span><span class="p">()</span>
|
|
<span class="n">localfile</span> <span class="o">=</span> <span class="n">url2pathname</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">stats</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">localfile</span><span class="p">)</span>
|
|
<span class="n">size</span> <span class="o">=</span> <span class="n">stats</span><span class="o">.</span><span class="n">st_size</span>
|
|
<span class="n">modified</span> <span class="o">=</span> <span class="n">email</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">formatdate</span><span class="p">(</span><span class="n">stats</span><span class="o">.</span><span class="n">st_mtime</span><span class="p">,</span> <span class="n">usegmt</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
<span class="n">mtype</span> <span class="o">=</span> <span class="n">mimetypes</span><span class="o">.</span><span class="n">guess_type</span><span class="p">(</span><span class="n">filename</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
|
<span class="n">headers</span> <span class="o">=</span> <span class="n">mimetools</span><span class="o">.</span><span class="n">Message</span><span class="p">(</span><span class="n">StringIO</span><span class="p">(</span>
|
|
<span class="s1">'Content-type: </span><span class="si">%s</span><span class="se">\n</span><span class="s1">Content-length: </span><span class="si">%d</span><span class="se">\n</span><span class="s1">Last-modified: </span><span class="si">%s</span><span class="se">\n</span><span class="s1">'</span> <span class="o">%</span>
|
|
<span class="p">(</span><span class="n">mtype</span> <span class="ow">or</span> <span class="s1">'text/plain'</span><span class="p">,</span> <span class="n">size</span><span class="p">,</span> <span class="n">modified</span><span class="p">)))</span>
|
|
<span class="k">if</span> <span class="n">host</span><span class="p">:</span>
|
|
<span class="n">host</span><span class="p">,</span> <span class="n">port</span> <span class="o">=</span> <span class="n">splitport</span><span class="p">(</span><span class="n">host</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">host</span> <span class="ow">or</span> \
|
|
<span class="p">(</span><span class="ow">not</span> <span class="n">port</span> <span class="ow">and</span> <span class="n">_safe_gethostbyname</span><span class="p">(</span><span class="n">host</span><span class="p">)</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_names</span><span class="p">()):</span>
|
|
<span class="k">if</span> <span class="n">host</span><span class="p">:</span>
|
|
<span class="n">origurl</span> <span class="o">=</span> <span class="s1">'file://'</span> <span class="o">+</span> <span class="n">host</span> <span class="o">+</span> <span class="n">filename</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">origurl</span> <span class="o">=</span> <span class="s1">'file://'</span> <span class="o">+</span> <span class="n">filename</span>
|
|
<span class="k">return</span> <span class="n">addinfourl</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="n">localfile</span><span class="p">,</span> <span class="s1">'rb'</span><span class="p">),</span> <span class="n">headers</span><span class="p">,</span> <span class="n">origurl</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">OSError</span><span class="p">,</span> <span class="n">msg</span><span class="p">:</span>
|
|
<span class="c1"># urllib2 users shouldn't expect OSErrors coming from urlopen()</span>
|
|
<span class="k">raise</span> <span class="n">URLError</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
|
|
<span class="k">raise</span> <span class="n">URLError</span><span class="p">(</span><span class="s1">'file not on local host'</span><span class="p">)</span>
|
|
|
|
<span class="k">class</span> <span class="nc">FTPHandler</span><span class="p">(</span><span class="n">BaseHandler</span><span class="p">):</span>
|
|
<span class="k">def</span> <span class="nf">ftp_open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">req</span><span class="p">):</span>
|
|
<span class="kn">import</span> <span class="nn">ftplib</span>
|
|
<span class="kn">import</span> <span class="nn">mimetypes</span>
|
|
<span class="n">host</span> <span class="o">=</span> <span class="n">req</span><span class="o">.</span><span class="n">get_host</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">host</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">URLError</span><span class="p">(</span><span class="s1">'ftp error: no host given'</span><span class="p">)</span>
|
|
<span class="n">host</span><span class="p">,</span> <span class="n">port</span> <span class="o">=</span> <span class="n">splitport</span><span class="p">(</span><span class="n">host</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">port</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">port</span> <span class="o">=</span> <span class="n">ftplib</span><span class="o">.</span><span class="n">FTP_PORT</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">port</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">port</span><span class="p">)</span>
|
|
|
|
<span class="c1"># username/password handling</span>
|
|
<span class="n">user</span><span class="p">,</span> <span class="n">host</span> <span class="o">=</span> <span class="n">splituser</span><span class="p">(</span><span class="n">host</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">user</span><span class="p">:</span>
|
|
<span class="n">user</span><span class="p">,</span> <span class="n">passwd</span> <span class="o">=</span> <span class="n">splitpasswd</span><span class="p">(</span><span class="n">user</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">passwd</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">host</span> <span class="o">=</span> <span class="n">unquote</span><span class="p">(</span><span class="n">host</span><span class="p">)</span>
|
|
<span class="n">user</span> <span class="o">=</span> <span class="n">user</span> <span class="ow">or</span> <span class="s1">''</span>
|
|
<span class="n">passwd</span> <span class="o">=</span> <span class="n">passwd</span> <span class="ow">or</span> <span class="s1">''</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">host</span> <span class="o">=</span> <span class="n">socket</span><span class="o">.</span><span class="n">gethostbyname</span><span class="p">(</span><span class="n">host</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">socket</span><span class="o">.</span><span class="n">error</span><span class="p">,</span> <span class="n">msg</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">URLError</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
|
|
<span class="n">path</span><span class="p">,</span> <span class="n">attrs</span> <span class="o">=</span> <span class="n">splitattr</span><span class="p">(</span><span class="n">req</span><span class="o">.</span><span class="n">get_selector</span><span class="p">())</span>
|
|
<span class="n">dirs</span> <span class="o">=</span> <span class="n">path</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)</span>
|
|
<span class="n">dirs</span> <span class="o">=</span> <span class="nb">map</span><span class="p">(</span><span class="n">unquote</span><span class="p">,</span> <span class="n">dirs</span><span class="p">)</span>
|
|
<span class="n">dirs</span><span class="p">,</span> <span class="n">file</span> <span class="o">=</span> <span class="n">dirs</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="n">dirs</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="n">dirs</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">dirs</span><span class="p">[</span><span class="mi">0</span><span class="p">]:</span>
|
|
<span class="n">dirs</span> <span class="o">=</span> <span class="n">dirs</span><span class="p">[</span><span class="mi">1</span><span class="p">:]</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">fw</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connect_ftp</span><span class="p">(</span><span class="n">user</span><span class="p">,</span> <span class="n">passwd</span><span class="p">,</span> <span class="n">host</span><span class="p">,</span> <span class="n">port</span><span class="p">,</span> <span class="n">dirs</span><span class="p">,</span> <span class="n">req</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
|
|
<span class="nb">type</span> <span class="o">=</span> <span class="n">file</span> <span class="ow">and</span> <span class="s1">'I'</span> <span class="ow">or</span> <span class="s1">'D'</span>
|
|
<span class="k">for</span> <span class="n">attr</span> <span class="ow">in</span> <span class="n">attrs</span><span class="p">:</span>
|
|
<span class="n">attr</span><span class="p">,</span> <span class="n">value</span> <span class="o">=</span> <span class="n">splitvalue</span><span class="p">(</span><span class="n">attr</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">attr</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'type'</span> <span class="ow">and</span> \
|
|
<span class="n">value</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'a'</span><span class="p">,</span> <span class="s1">'A'</span><span class="p">,</span> <span class="s1">'i'</span><span class="p">,</span> <span class="s1">'I'</span><span class="p">,</span> <span class="s1">'d'</span><span class="p">,</span> <span class="s1">'D'</span><span class="p">):</span>
|
|
<span class="nb">type</span> <span class="o">=</span> <span class="n">value</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
|
|
<span class="n">fp</span><span class="p">,</span> <span class="n">retrlen</span> <span class="o">=</span> <span class="n">fw</span><span class="o">.</span><span class="n">retrfile</span><span class="p">(</span><span class="n">file</span><span class="p">,</span> <span class="nb">type</span><span class="p">)</span>
|
|
<span class="n">headers</span> <span class="o">=</span> <span class="s2">""</span>
|
|
<span class="n">mtype</span> <span class="o">=</span> <span class="n">mimetypes</span><span class="o">.</span><span class="n">guess_type</span><span class="p">(</span><span class="n">req</span><span class="o">.</span><span class="n">get_full_url</span><span class="p">())[</span><span class="mi">0</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="n">mtype</span><span class="p">:</span>
|
|
<span class="n">headers</span> <span class="o">+=</span> <span class="s2">"Content-type: </span><span class="si">%s</span><span class="se">\n</span><span class="s2">"</span> <span class="o">%</span> <span class="n">mtype</span>
|
|
<span class="k">if</span> <span class="n">retrlen</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">retrlen</span> <span class="o">>=</span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="n">headers</span> <span class="o">+=</span> <span class="s2">"Content-length: </span><span class="si">%d</span><span class="se">\n</span><span class="s2">"</span> <span class="o">%</span> <span class="n">retrlen</span>
|
|
<span class="n">sf</span> <span class="o">=</span> <span class="n">StringIO</span><span class="p">(</span><span class="n">headers</span><span class="p">)</span>
|
|
<span class="n">headers</span> <span class="o">=</span> <span class="n">mimetools</span><span class="o">.</span><span class="n">Message</span><span class="p">(</span><span class="n">sf</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">addinfourl</span><span class="p">(</span><span class="n">fp</span><span class="p">,</span> <span class="n">headers</span><span class="p">,</span> <span class="n">req</span><span class="o">.</span><span class="n">get_full_url</span><span class="p">())</span>
|
|
<span class="k">except</span> <span class="n">ftplib</span><span class="o">.</span><span class="n">all_errors</span><span class="p">,</span> <span class="n">msg</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">URLError</span><span class="p">,</span> <span class="p">(</span><span class="s1">'ftp error: </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">msg</span><span class="p">),</span> <span class="n">sys</span><span class="o">.</span><span class="n">exc_info</span><span class="p">()[</span><span class="mi">2</span><span class="p">]</span>
|
|
|
|
<span class="k">def</span> <span class="nf">connect_ftp</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">user</span><span class="p">,</span> <span class="n">passwd</span><span class="p">,</span> <span class="n">host</span><span class="p">,</span> <span class="n">port</span><span class="p">,</span> <span class="n">dirs</span><span class="p">,</span> <span class="n">timeout</span><span class="p">):</span>
|
|
<span class="n">fw</span> <span class="o">=</span> <span class="n">ftpwrapper</span><span class="p">(</span><span class="n">user</span><span class="p">,</span> <span class="n">passwd</span><span class="p">,</span> <span class="n">host</span><span class="p">,</span> <span class="n">port</span><span class="p">,</span> <span class="n">dirs</span><span class="p">,</span> <span class="n">timeout</span><span class="p">,</span>
|
|
<span class="n">persistent</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
|
<span class="c1">## fw.ftp.set_debuglevel(1)</span>
|
|
<span class="k">return</span> <span class="n">fw</span>
|
|
|
|
<span class="k">class</span> <span class="nc">CacheFTPHandler</span><span class="p">(</span><span class="n">FTPHandler</span><span class="p">):</span>
|
|
<span class="c1"># XXX would be nice to have pluggable cache strategies</span>
|
|
<span class="c1"># XXX this stuff is definitely not thread safe</span>
|
|
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">cache</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">soonest</span> <span class="o">=</span> <span class="mi">0</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">delay</span> <span class="o">=</span> <span class="mi">60</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">max_conns</span> <span class="o">=</span> <span class="mi">16</span>
|
|
|
|
<span class="k">def</span> <span class="nf">setTimeout</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">t</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">delay</span> <span class="o">=</span> <span class="n">t</span>
|
|
|
|
<span class="k">def</span> <span class="nf">setMaxConns</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">m</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">max_conns</span> <span class="o">=</span> <span class="n">m</span>
|
|
|
|
<span class="k">def</span> <span class="nf">connect_ftp</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">user</span><span class="p">,</span> <span class="n">passwd</span><span class="p">,</span> <span class="n">host</span><span class="p">,</span> <span class="n">port</span><span class="p">,</span> <span class="n">dirs</span><span class="p">,</span> <span class="n">timeout</span><span class="p">):</span>
|
|
<span class="n">key</span> <span class="o">=</span> <span class="n">user</span><span class="p">,</span> <span class="n">host</span><span class="p">,</span> <span class="n">port</span><span class="p">,</span> <span class="s1">'/'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">dirs</span><span class="p">),</span> <span class="n">timeout</span>
|
|
<span class="k">if</span> <span class="n">key</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cache</span><span class="p">:</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">delay</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">cache</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">ftpwrapper</span><span class="p">(</span><span class="n">user</span><span class="p">,</span> <span class="n">passwd</span><span class="p">,</span> <span class="n">host</span><span class="p">,</span> <span class="n">port</span><span class="p">,</span> <span class="n">dirs</span><span class="p">,</span> <span class="n">timeout</span><span class="p">)</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">delay</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">check_cache</span><span class="p">()</span>
|
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">cache</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
|
|
|
|
<span class="k">def</span> <span class="nf">check_cache</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="c1"># first check for old ones</span>
|
|
<span class="n">t</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">soonest</span> <span class="o"><=</span> <span class="n">t</span><span class="p">:</span>
|
|
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
|
<span class="k">if</span> <span class="n">v</span> <span class="o"><</span> <span class="n">t</span><span class="p">:</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">cache</span><span class="p">[</span><span class="n">k</span><span class="p">]</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
|
<span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">cache</span><span class="p">[</span><span class="n">k</span><span class="p">]</span>
|
|
<span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">[</span><span class="n">k</span><span class="p">]</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">soonest</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="o">.</span><span class="n">values</span><span class="p">())</span>
|
|
|
|
<span class="c1"># then check the size</span>
|
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cache</span><span class="p">)</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_conns</span><span class="p">:</span>
|
|
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
|
<span class="k">if</span> <span class="n">v</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">soonest</span><span class="p">:</span>
|
|
<span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">cache</span><span class="p">[</span><span class="n">k</span><span class="p">]</span>
|
|
<span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">[</span><span class="n">k</span><span class="p">]</span>
|
|
<span class="k">break</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">soonest</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="o">.</span><span class="n">values</span><span class="p">())</span>
|
|
|
|
<span class="k">def</span> <span class="nf">clear_cache</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="k">for</span> <span class="n">conn</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cache</span><span class="o">.</span><span class="n">values</span><span class="p">():</span>
|
|
<span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">cache</span><span class="o">.</span><span class="n">clear</span><span class="p">()</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="o">.</span><span class="n">clear</span><span class="p">()</span>
|
|
</pre></div>
|
|
|
|
</div>
|
|
</div>
|
|
<footer>
|
|
|
|
|
|
<hr/>
|
|
|
|
<div role="contentinfo">
|
|
<p>
|
|
© Copyright 2015-2019, Vector 35 Inc.
|
|
|
|
</p>
|
|
</div>
|
|
|
|
</footer>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</section>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
|
|
<script type="text/javascript" src="../_static/jquery.js"></script>
|
|
<script type="text/javascript" src="../_static/underscore.js"></script>
|
|
<script type="text/javascript" src="../_static/doctools.js"></script>
|
|
<script type="text/javascript" src="../_static/language_data.js"></script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<script type="text/javascript" src="../_static/js/theme.js"></script>
|
|
|
|
|
|
|
|
|
|
<script type="text/javascript">
|
|
jQuery(function () {
|
|
SphinxRtdTheme.StickyNav.enable();
|
|
});
|
|
</script>
|
|
|
|
|
|
</body>
|
|
</html> |