--- /dev/null
+Tornado
+=======
+Tornado is an open source version of the scalable, non-blocking web server
+and and tools that power FriendFeed. Documentation and downloads are
+available at http://www.tornadoweb.org/
+
+Tornado is licensed under the Apache Licence, Version 2.0
+(http://www.apache.org/licenses/LICENSE-2.0.html).
+
+Installation
+============
+To install:
+
+ python setup.py build
+ sudo python setup.py install
+
+Tornado has been tested on Python 2.5 and 2.6. To use all of the features
+of Tornado, you need to have PycURL and a JSON library like simplejson
+installed.
+
+On Mac OS X, you can install the packages with:
+
+ sudo easy_install setuptools pycurl==7.16.2.1 simplejson
+
+On Ubuntu Linux, you can install the packages with:
+
+ sudo apt-get install python-pycurl python-simplejson
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""The Tornado web server and tools."""
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Implementations of various third-party authentication schemes.
+
+All the classes in this file are class Mixins designed to be used with
+web.py RequestHandler classes. The primary methods for each service are
+authenticate_redirect(), authorize_redirect(), and get_authenticated_user().
+The former should be called to redirect the user to, e.g., the OpenID
+authentication page on the third party service, and the latter should
+be called upon return to get the user data from the data returned by
+the third party service.
+
+They all take slightly different arguments due to the fact all these
+services implement authentication and authorization slightly differently.
+See the individual service classes below for complete documentation.
+
+Example usage for Google OpenID:
+
+class GoogleHandler(tornado.web.RequestHandler, tornado.auth.GoogleMixin):
+ @tornado.web.asynchronous
+ def get(self):
+ if self.get_argument("openid.mode", None):
+ self.get_authenticated_user(self.async_callback(self._on_auth))
+ return
+ self.authenticate_redirect()
+
+ def _on_auth(self, user):
+ if not user:
+ raise tornado.web.HTTPError(500, "Google auth failed")
+ # Save the user with, e.g., set_secure_cookie()
+
+"""
+
+import binascii
+import cgi
+import hashlib
+import hmac
+import httpclient
+import escape
+import logging
+import time
+import urllib
+import urlparse
+import uuid
+
+class OpenIdMixin(object):
+ """Abstract implementation of OpenID and Attribute Exchange.
+
+ See GoogleMixin below for example implementations.
+ """
+ def authenticate_redirect(self, callback_uri=None,
+ ax_attrs=["name","email","language","username"]):
+ """Returns the authentication URL for this service.
+
+ After authentication, the service will redirect back to the given
+ callback URI.
+
+ We request the given attributes for the authenticated user by
+ default (name, email, language, and username). If you don't need
+ all those attributes for your app, you can request fewer with
+ the ax_attrs keyword argument.
+ """
+ callback_uri = callback_uri or self.request.path
+ args = self._openid_args(callback_uri, ax_attrs=ax_attrs)
+ self.redirect(self._OPENID_ENDPOINT + "?" + urllib.urlencode(args))
+
+ def get_authenticated_user(self, callback):
+ """Fetches the authenticated user data upon redirect.
+
+ This method should be called by the handler that receives the
+ redirect from the authenticate_redirect() or authorize_redirect()
+ methods.
+ """
+ # Verify the OpenID response via direct request to the OP
+ args = dict((k, v[-1]) for k, v in self.request.arguments.iteritems())
+ args["openid.mode"] = u"check_authentication"
+ url = self._OPENID_ENDPOINT + "?" + urllib.urlencode(args)
+ http = httpclient.AsyncHTTPClient()
+ http.fetch(url, self.async_callback(
+ self._on_authentication_verified, callback))
+
+ def _openid_args(self, callback_uri, ax_attrs=[], oauth_scope=None):
+ url = urlparse.urljoin(self.request.full_url(), callback_uri)
+ args = {
+ "openid.ns": "http://specs.openid.net/auth/2.0",
+ "openid.claimed_id":
+ "http://specs.openid.net/auth/2.0/identifier_select",
+ "openid.identity":
+ "http://specs.openid.net/auth/2.0/identifier_select",
+ "openid.return_to": url,
+ "openid.realm": self.request.protocol + "://" + self.request.host + "/",
+ "openid.mode": "checkid_setup",
+ }
+ if ax_attrs:
+ args.update({
+ "openid.ns.ax": "http://openid.net/srv/ax/1.0",
+ "openid.ax.mode": "fetch_request",
+ })
+ ax_attrs = set(ax_attrs)
+ required = []
+ if "name" in ax_attrs:
+ ax_attrs -= set(["name", "firstname", "fullname", "lastname"])
+ required += ["firstname", "fullname", "lastname"]
+ args.update({
+ "openid.ax.type.firstname":
+ "http://axschema.org/namePerson/first",
+ "openid.ax.type.fullname":
+ "http://axschema.org/namePerson",
+ "openid.ax.type.lastname":
+ "http://axschema.org/namePerson/last",
+ })
+ known_attrs = {
+ "email": "http://axschema.org/contact/email",
+ "language": "http://axschema.org/pref/language",
+ "username": "http://axschema.org/namePerson/friendly",
+ }
+ for name in ax_attrs:
+ args["openid.ax.type." + name] = known_attrs[name]
+ required.append(name)
+ args["openid.ax.required"] = ",".join(required)
+ if oauth_scope:
+ args.update({
+ "openid.ns.oauth":
+ "http://specs.openid.net/extensions/oauth/1.0",
+ "openid.oauth.consumer": self.request.host.split(":")[0],
+ "openid.oauth.scope": oauth_scope,
+ })
+ return args
+
+ def _on_authentication_verified(self, callback, response):
+ if response.error or u"is_valid:true" not in response.body:
+ logging.warning("Invalid OpenID response: %s", response.error or
+ response.body)
+ callback(None)
+ return
+
+ # Make sure we got back at least an email from attribute exchange
+ ax_ns = None
+ for name, values in self.request.arguments.iteritems():
+ if name.startswith("openid.ns.") and \
+ values[-1] == u"http://openid.net/srv/ax/1.0":
+ ax_ns = name[10:]
+ break
+ def get_ax_arg(uri):
+ if not ax_ns: return u""
+ prefix = "openid." + ax_ns + ".type."
+ ax_name = None
+ for name, values in self.request.arguments.iteritems():
+ if values[-1] == uri and name.startswith(prefix):
+ part = name[len(prefix):]
+ ax_name = "openid." + ax_ns + ".value." + part
+ break
+ if not ax_name: return u""
+ return self.get_argument(ax_name, u"")
+
+ email = get_ax_arg("http://axschema.org/contact/email")
+ name = get_ax_arg("http://axschema.org/namePerson")
+ first_name = get_ax_arg("http://axschema.org/namePerson/first")
+ last_name = get_ax_arg("http://axschema.org/namePerson/last")
+ username = get_ax_arg("http://axschema.org/namePerson/friendly")
+ locale = get_ax_arg("http://axschema.org/pref/language").lower()
+ user = dict()
+ name_parts = []
+ if first_name:
+ user["first_name"] = first_name
+ name_parts.append(first_name)
+ if last_name:
+ user["last_name"] = last_name
+ name_parts.append(last_name)
+ if name:
+ user["name"] = name
+ elif name_parts:
+ user["name"] = u" ".join(name_parts)
+ elif email:
+ user["name"] = email.split("@")[0]
+ if email: user["email"] = email
+ if locale: user["locale"] = locale
+ if username: user["username"] = username
+ callback(user)
+
+
+class OAuthMixin(object):
+ """Abstract implementation of OAuth.
+
+ See TwitterMixin and FriendFeedMixin below for example implementations.
+ """
+ def authorize_redirect(self, callback_uri=None):
+ """Redirects the user to obtain OAuth authorization for this service.
+
+ Twitter and FriendFeed both require that you register a Callback
+ URL with your application. You should call this method to log the
+ user in, and then call get_authenticated_user() in the handler
+ you registered as your Callback URL to complete the authorization
+ process.
+
+ This method sets a cookie called _oauth_request_token which is
+ subsequently used (and cleared) in get_authenticated_user for
+ security purposes.
+ """
+ if callback_uri and getattr(self, "_OAUTH_NO_CALLBACKS", False):
+ raise Exception("This service does not support oauth_callback")
+ http = httpclient.AsyncHTTPClient()
+ http.fetch(self._oauth_request_token_url(), self.async_callback(
+ self._on_request_token, self._OAUTH_AUTHORIZE_URL, callback_uri))
+
+ def get_authenticated_user(self, callback):
+ """Gets the OAuth authorized user and access token on callback.
+
+ This method should be called from the handler for your registered
+ OAuth Callback URL to complete the registration process. We call
+ callback with the authenticated user, which in addition to standard
+ attributes like 'name' includes the 'access_key' attribute, which
+ contains the OAuth access you can use to make authorized requests
+ to this service on behalf of the user.
+ """
+ request_key = self.get_argument("oauth_token")
+ request_cookie = self.get_cookie("_oauth_request_token")
+ if not request_cookie:
+ logging.warning("Missing OAuth request token cookie")
+ callback(None)
+ return
+ cookie_key, cookie_secret = request_cookie.split("|")
+ if cookie_key != request_key:
+ logging.warning("Request token does not match cookie")
+ callback(None)
+ return
+ token = dict(key=cookie_key, secret=cookie_secret)
+ http = httpclient.AsyncHTTPClient()
+ http.fetch(self._oauth_access_token_url(token), self.async_callback(
+ self._on_access_token, callback))
+
+ def _oauth_request_token_url(self):
+ consumer_token = self._oauth_consumer_token()
+ url = self._OAUTH_REQUEST_TOKEN_URL
+ args = dict(
+ oauth_consumer_key=consumer_token["key"],
+ oauth_signature_method="HMAC-SHA1",
+ oauth_timestamp=str(int(time.time())),
+ oauth_nonce=binascii.b2a_hex(uuid.uuid4().bytes),
+ oauth_version="1.0",
+ )
+ signature = _oauth_signature(consumer_token, "GET", url, args)
+ args["oauth_signature"] = signature
+ return url + "?" + urllib.urlencode(args)
+
+ def _on_request_token(self, authorize_url, callback_uri, response):
+ if response.error:
+ raise Exception("Could not get request token")
+ request_token = _oauth_parse_response(response.body)
+ data = "|".join([request_token["key"], request_token["secret"]])
+ self.set_cookie("_oauth_request_token", data)
+ args = dict(oauth_token=request_token["key"])
+ if callback_uri:
+ args["oauth_callback"] = urlparse.urljoin(
+ self.request.full_url(), callback_uri)
+ self.redirect(authorize_url + "?" + urllib.urlencode(args))
+
+ def _oauth_access_token_url(self, request_token):
+ consumer_token = self._oauth_consumer_token()
+ url = self._OAUTH_ACCESS_TOKEN_URL
+ args = dict(
+ oauth_consumer_key=consumer_token["key"],
+ oauth_token=request_token["key"],
+ oauth_signature_method="HMAC-SHA1",
+ oauth_timestamp=str(int(time.time())),
+ oauth_nonce=binascii.b2a_hex(uuid.uuid4().bytes),
+ oauth_version="1.0",
+ )
+ signature = _oauth_signature(consumer_token, "GET", url, args,
+ request_token)
+ args["oauth_signature"] = signature
+ return url + "?" + urllib.urlencode(args)
+
+ def _on_access_token(self, callback, response):
+ if response.error:
+ logging.warning("Could not fetch access token")
+ callback(None)
+ return
+ access_token = _oauth_parse_response(response.body)
+ user = self._oauth_get_user(access_token, self.async_callback(
+ self._on_oauth_get_user, access_token, callback))
+
+ def _oauth_get_user(self, access_token, callback):
+ raise NotImplementedError()
+
+ def _on_oauth_get_user(self, access_token, callback, user):
+ if not user:
+ callback(None)
+ return
+ user["access_token"] = access_token
+ callback(user)
+
+ def _oauth_request_parameters(self, url, access_token, parameters={},
+ method="GET"):
+ """Returns the OAuth parameters as a dict for the given request.
+
+ parameters should include all POST arguments and query string arguments
+ that will be sent with the request.
+ """
+ consumer_token = self._oauth_consumer_token()
+ base_args = dict(
+ oauth_consumer_key=consumer_token["key"],
+ oauth_token=access_token["key"],
+ oauth_signature_method="HMAC-SHA1",
+ oauth_timestamp=str(int(time.time())),
+ oauth_nonce=binascii.b2a_hex(uuid.uuid4().bytes),
+ oauth_version="1.0",
+ )
+ args = {}
+ args.update(base_args)
+ args.update(parameters)
+ signature = _oauth_signature(consumer_token, method, url, args,
+ access_token)
+ base_args["oauth_signature"] = signature
+ return base_args
+
+
+class TwitterMixin(OAuthMixin):
+ """Twitter OAuth authentication.
+
+ To authenticate with Twitter, register your application with
+ Twitter at http://twitter.com/apps. Then copy your Consumer Key and
+ Consumer Secret to the application settings 'twitter_consumer_key' and
+ 'twitter_consumer_secret'. Use this Mixin on the handler for the URL
+ you registered as your application's Callback URL.
+
+ When your application is set up, you can use this Mixin like this
+ to authenticate the user with Twitter and get access to their stream:
+
+ class TwitterHandler(tornado.web.RequestHandler,
+ tornado.auth.TwitterMixin):
+ @tornado.web.asynchronous
+ def get(self):
+ if self.get_argument("oauth_token", None):
+ self.get_authenticated_user(self.async_callback(self._on_auth))
+ return
+ self.authorize_redirect()
+
+ def _on_auth(self, user):
+ if not user:
+ raise tornado.web.HTTPError(500, "Twitter auth failed")
+ # Save the user using, e.g., set_secure_cookie()
+
+ The user object returned by get_authenticated_user() includes the
+ attributes 'username', 'name', and all of the custom Twitter user
+ attributes describe at
+ http://apiwiki.twitter.com/Twitter-REST-API-Method%3A-users%C2%A0show
+ in addition to 'access_token'. You should save the access token with
+ the user; it is required to make requests on behalf of the user later
+ with twitter_request().
+ """
+ _OAUTH_REQUEST_TOKEN_URL = "http://api.twitter.com/oauth/request_token"
+ _OAUTH_ACCESS_TOKEN_URL = "http://api.twitter.com/oauth/access_token"
+ _OAUTH_AUTHORIZE_URL = "http://api.twitter.com/oauth/authorize"
+ _OAUTH_AUTHENTICATE_URL = "http://api.twitter.com/oauth/authenticate"
+ _OAUTH_NO_CALLBACKS = True
+
+ def authenticate_redirect(self):
+ """Just like authorize_redirect(), but auto-redirects if authorized.
+
+ This is generally the right interface to use if you are using
+ Twitter for single-sign on.
+ """
+ http = httpclient.AsyncHTTPClient()
+ http.fetch(self._oauth_request_token_url(), self.async_callback(
+ self._on_request_token, self._OAUTH_AUTHENTICATE_URL, None))
+
+ def twitter_request(self, path, callback, access_token=None,
+ post_args=None, **args):
+ """Fetches the given API path, e.g., "/statuses/user_timeline/btaylor"
+
+ The path should not include the format (we automatically append
+ ".json" and parse the JSON output).
+
+ If the request is a POST, post_args should be provided. Query
+ string arguments should be given as keyword arguments.
+
+ All the Twitter methods are documented at
+ http://apiwiki.twitter.com/Twitter-API-Documentation.
+
+ Many methods require an OAuth access token which you can obtain
+ through authorize_redirect() and get_authenticated_user(). The
+ user returned through that process includes an 'access_token'
+ attribute that can be used to make authenticated requests via
+ this method. Example usage:
+
+ class MainHandler(tornado.web.RequestHandler,
+ tornado.auth.TwitterMixin):
+ @tornado.web.authenticated
+ @tornado.web.asynchronous
+ def get(self):
+ self.twitter_request(
+ "/statuses/update",
+ post_args={"status": "Testing Tornado Web Server"},
+ access_token=user["access_token"],
+ callback=self.async_callback(self._on_post))
+
+ def _on_post(self, new_entry):
+ if not new_entry:
+ # Call failed; perhaps missing permission?
+ self.authorize_redirect()
+ return
+ self.finish("Posted a message!")
+
+ """
+ # Add the OAuth resource request signature if we have credentials
+ url = "http://api.twitter.com/1" + path + ".json"
+ if access_token:
+ all_args = {}
+ all_args.update(args)
+ all_args.update(post_args or {})
+ consumer_token = self._oauth_consumer_token()
+ method = "POST" if post_args is not None else "GET"
+ oauth = self._oauth_request_parameters(
+ url, access_token, all_args, method=method)
+ args.update(oauth)
+ if args: url += "?" + urllib.urlencode(args)
+ callback = self.async_callback(self._on_twitter_request, callback)
+ http = httpclient.AsyncHTTPClient()
+ if post_args is not None:
+ http.fetch(url, method="POST", body=urllib.urlencode(post_args),
+ callback=callback)
+ else:
+ http.fetch(url, callback=callback)
+
+ def _on_twitter_request(self, callback, response):
+ if response.error:
+ logging.warning("Error response %s fetching %s", response.error,
+ response.request.url)
+ callback(None)
+ return
+ callback(escape.json_decode(response.body))
+
+ def _oauth_consumer_token(self):
+ self.require_setting("twitter_consumer_key", "Twitter OAuth")
+ self.require_setting("twitter_consumer_secret", "Twitter OAuth")
+ return dict(
+ key=self.settings["twitter_consumer_key"],
+ secret=self.settings["twitter_consumer_secret"])
+
+ def _oauth_get_user(self, access_token, callback):
+ callback = self.async_callback(self._parse_user_response, callback)
+ self.twitter_request(
+ "/users/show/" + access_token["screen_name"],
+ access_token=access_token, callback=callback)
+
+ def _parse_user_response(self, callback, user):
+ if user:
+ user["username"] = user["screen_name"]
+ callback(user)
+
+
+class FriendFeedMixin(OAuthMixin):
+ """FriendFeed OAuth authentication.
+
+ To authenticate with FriendFeed, register your application with
+ FriendFeed at http://friendfeed.com/api/applications. Then
+ copy your Consumer Key and Consumer Secret to the application settings
+ 'friendfeed_consumer_key' and 'friendfeed_consumer_secret'. Use
+ this Mixin on the handler for the URL you registered as your
+ application's Callback URL.
+
+ When your application is set up, you can use this Mixin like this
+ to authenticate the user with FriendFeed and get access to their feed:
+
+ class FriendFeedHandler(tornado.web.RequestHandler,
+ tornado.auth.FriendFeedMixin):
+ @tornado.web.asynchronous
+ def get(self):
+ if self.get_argument("oauth_token", None):
+ self.get_authenticated_user(self.async_callback(self._on_auth))
+ return
+ self.authorize_redirect()
+
+ def _on_auth(self, user):
+ if not user:
+ raise tornado.web.HTTPError(500, "FriendFeed auth failed")
+ # Save the user using, e.g., set_secure_cookie()
+
+ The user object returned by get_authenticated_user() includes the
+ attributes 'username', 'name', and 'description' in addition to
+ 'access_token'. You should save the access token with the user;
+ it is required to make requests on behalf of the user later with
+ friendfeed_request().
+ """
+ _OAUTH_REQUEST_TOKEN_URL = "https://friendfeed.com/account/oauth/request_token"
+ _OAUTH_ACCESS_TOKEN_URL = "https://friendfeed.com/account/oauth/access_token"
+ _OAUTH_AUTHORIZE_URL = "https://friendfeed.com/account/oauth/authorize"
+ _OAUTH_NO_CALLBACKS = True
+
+ def friendfeed_request(self, path, callback, access_token=None,
+ post_args=None, **args):
+ """Fetches the given relative API path, e.g., "/bret/friends"
+
+ If the request is a POST, post_args should be provided. Query
+ string arguments should be given as keyword arguments.
+
+ All the FriendFeed methods are documented at
+ http://friendfeed.com/api/documentation.
+
+ Many methods require an OAuth access token which you can obtain
+ through authorize_redirect() and get_authenticated_user(). The
+ user returned through that process includes an 'access_token'
+ attribute that can be used to make authenticated requests via
+ this method. Example usage:
+
+ class MainHandler(tornado.web.RequestHandler,
+ tornado.auth.FriendFeedMixin):
+ @tornado.web.authenticated
+ @tornado.web.asynchronous
+ def get(self):
+ self.friendfeed_request(
+ "/entry",
+ post_args={"body": "Testing Tornado Web Server"},
+ access_token=self.current_user["access_token"],
+ callback=self.async_callback(self._on_post))
+
+ def _on_post(self, new_entry):
+ if not new_entry:
+ # Call failed; perhaps missing permission?
+ self.authorize_redirect()
+ return
+ self.finish("Posted a message!")
+
+ """
+ # Add the OAuth resource request signature if we have credentials
+ url = "http://friendfeed-api.com/v2" + path
+ if access_token:
+ all_args = {}
+ all_args.update(args)
+ all_args.update(post_args or {})
+ consumer_token = self._oauth_consumer_token()
+ method = "POST" if post_args is not None else "GET"
+ oauth = self._oauth_request_parameters(
+ url, access_token, all_args, method=method)
+ args.update(oauth)
+ if args: url += "?" + urllib.urlencode(args)
+ callback = self.async_callback(self._on_friendfeed_request, callback)
+ http = httpclient.AsyncHTTPClient()
+ if post_args is not None:
+ http.fetch(url, method="POST", body=urllib.urlencode(post_args),
+ callback=callback)
+ else:
+ http.fetch(url, callback=callback)
+
+ def _on_friendfeed_request(self, callback, response):
+ if response.error:
+ logging.warning("Error response %s fetching %s", response.error,
+ response.request.url)
+ callback(None)
+ return
+ callback(escape.json_decode(response.body))
+
+ def _oauth_consumer_token(self):
+ self.require_setting("friendfeed_consumer_key", "FriendFeed OAuth")
+ self.require_setting("friendfeed_consumer_secret", "FriendFeed OAuth")
+ return dict(
+ key=self.settings["friendfeed_consumer_key"],
+ secret=self.settings["friendfeed_consumer_secret"])
+
+ def _oauth_get_user(self, access_token, callback):
+ callback = self.async_callback(self._parse_user_response, callback)
+ self.friendfeed_request(
+ "/feedinfo/" + access_token["username"],
+ include="id,name,description", access_token=access_token,
+ callback=callback)
+
+ def _parse_user_response(self, callback, user):
+ if user:
+ user["username"] = user["id"]
+ callback(user)
+
+
+class GoogleMixin(OpenIdMixin, OAuthMixin):
+ """Google Open ID / OAuth authentication.
+
+ No application registration is necessary to use Google for authentication
+ or to access Google resources on behalf of a user. To authenticate with
+ Google, redirect with authenticate_redirect(). On return, parse the
+ response with get_authenticated_user(). We send a dict containing the
+ values for the user, including 'email', 'name', and 'locale'.
+ Example usage:
+
+ class GoogleHandler(tornado.web.RequestHandler, tornado.auth.GoogleMixin):
+ @tornado.web.asynchronous
+ def get(self):
+ if self.get_argument("openid.mode", None):
+ self.get_authenticated_user(self.async_callback(self._on_auth))
+ return
+ self.authenticate_redirect()
+
+ def _on_auth(self, user):
+ if not user:
+ raise tornado.web.HTTPError(500, "Google auth failed")
+ # Save the user with, e.g., set_secure_cookie()
+
+ """
+ _OPENID_ENDPOINT = "https://www.google.com/accounts/o8/ud"
+ _OAUTH_ACCESS_TOKEN_URL = "https://www.google.com/accounts/OAuthGetAccessToken"
+
+ def authorize_redirect(self, oauth_scope, callback_uri=None,
+ ax_attrs=["name","email","language","username"]):
+ """Authenticates and authorizes for the given Google resource.
+
+ Some of the available resources are:
+
+ Gmail Contacts - http://www.google.com/m8/feeds/
+ Calendar - http://www.google.com/calendar/feeds/
+ Finance - http://finance.google.com/finance/feeds/
+
+ You can authorize multiple resources by separating the resource
+ URLs with a space.
+ """
+ callback_uri = callback_uri or self.request.path
+ args = self._openid_args(callback_uri, ax_attrs=ax_attrs,
+ oauth_scope=oauth_scope)
+ self.redirect(self._OPENID_ENDPOINT + "?" + urllib.urlencode(args))
+
+ def get_authenticated_user(self, callback):
+ """Fetches the authenticated user data upon redirect."""
+ # Look to see if we are doing combined OpenID/OAuth
+ oauth_ns = ""
+ for name, values in self.request.arguments.iteritems():
+ if name.startswith("openid.ns.") and \
+ values[-1] == u"http://specs.openid.net/extensions/oauth/1.0":
+ oauth_ns = name[10:]
+ break
+ token = self.get_argument("openid." + oauth_ns + ".request_token", "")
+ if token:
+ http = httpclient.AsyncHTTPClient()
+ token = dict(key=token, secret="")
+ http.fetch(self._oauth_access_token_url(token),
+ self.async_callback(self._on_access_token, callback))
+ else:
+ OpenIdMixin.get_authenticated_user(self, callback)
+
+ def _oauth_consumer_token(self):
+ self.require_setting("google_consumer_key", "Google OAuth")
+ self.require_setting("google_consumer_secret", "Google OAuth")
+ return dict(
+ key=self.settings["google_consumer_key"],
+ secret=self.settings["google_consumer_secret"])
+
+ def _oauth_get_user(self, access_token, callback):
+ OpenIdMixin.get_authenticated_user(self, callback)
+
+
+class FacebookMixin(object):
+ """Facebook Connect authentication.
+
+ To authenticate with Facebook, register your application with
+ Facebook at http://www.facebook.com/developers/apps.php. Then
+ copy your API Key and Application Secret to the application settings
+ 'facebook_api_key' and 'facebook_secret'.
+
+ When your application is set up, you can use this Mixin like this
+ to authenticate the user with Facebook:
+
+ class FacebookHandler(tornado.web.RequestHandler,
+ tornado.auth.FacebookMixin):
+ @tornado.web.asynchronous
+ def get(self):
+ if self.get_argument("session", None):
+ self.get_authenticated_user(self.async_callback(self._on_auth))
+ return
+ self.authenticate_redirect()
+
+ def _on_auth(self, user):
+ if not user:
+ raise tornado.web.HTTPError(500, "Facebook auth failed")
+ # Save the user using, e.g., set_secure_cookie()
+
+ The user object returned by get_authenticated_user() includes the
+ attributes 'facebook_uid' and 'name' in addition to session attributes
+ like 'session_key'. You should save the session key with the user; it is
+ required to make requests on behalf of the user later with
+ facebook_request().
+ """
+ def authenticate_redirect(self, callback_uri=None, cancel_uri=None,
+ extended_permissions=None):
+ """Authenticates/installs this app for the current user."""
+ self.require_setting("facebook_api_key", "Facebook Connect")
+ callback_uri = callback_uri or self.request.path
+ args = {
+ "api_key": self.settings["facebook_api_key"],
+ "v": "1.0",
+ "fbconnect": "true",
+ "display": "page",
+ "next": urlparse.urljoin(self.request.full_url(), callback_uri),
+ "return_session": "true",
+ }
+ if cancel_uri:
+ args["cancel_url"] = urlparse.urljoin(
+ self.request.full_url(), cancel_uri)
+ if extended_permissions:
+ if isinstance(extended_permissions, basestring):
+ extended_permissions = [extended_permissions]
+ args["req_perms"] = ",".join(extended_permissions)
+ self.redirect("http://www.facebook.com/login.php?" +
+ urllib.urlencode(args))
+
+ def authorize_redirect(self, extended_permissions, callback_uri=None,
+ cancel_uri=None):
+ """Redirects to an authorization request for the given FB resource.
+
+ The available resource names are listed at
+ http://wiki.developers.facebook.com/index.php/Extended_permission.
+ The most common resource types include:
+
+ publish_stream
+ read_stream
+ email
+ sms
+
+ extended_permissions can be a single permission name or a list of
+ names. To get the session secret and session key, call
+ get_authenticated_user() just as you would with
+ authenticate_redirect().
+ """
+ self.authenticate_redirect(callback_uri, cancel_uri,
+ extended_permissions)
+
+ def get_authenticated_user(self, callback):
+ """Fetches the authenticated Facebook user.
+
+ The authenticated user includes the special Facebook attributes
+ 'session_key' and 'facebook_uid' in addition to the standard
+ user attributes like 'name'.
+ """
+ self.require_setting("facebook_api_key", "Facebook Connect")
+ session = escape.json_decode(self.get_argument("session"))
+ self.facebook_request(
+ method="facebook.users.getInfo",
+ callback=self.async_callback(
+ self._on_get_user_info, callback, session),
+ session_key=session["session_key"],
+ uids=session["uid"],
+ fields="uid,first_name,last_name,name,locale,pic_square," \
+ "profile_url,username")
+
+ def facebook_request(self, method, callback, **args):
+ """Makes a Facebook API REST request.
+
+ We automatically include the Facebook API key and signature, but
+ it is the callers responsibility to include 'session_key' and any
+ other required arguments to the method.
+
+ The available Facebook methods are documented here:
+ http://wiki.developers.facebook.com/index.php/API
+
+ Here is an example for the stream.get() method:
+
+ class MainHandler(tornado.web.RequestHandler,
+ tornado.auth.FacebookMixin):
+ @tornado.web.authenticated
+ @tornado.web.asynchronous
+ def get(self):
+ self.facebook_request(
+ method="stream.get",
+ callback=self.async_callback(self._on_stream),
+ session_key=self.current_user["session_key"])
+
+ def _on_stream(self, stream):
+ if stream is None:
+ # Not authorized to read the stream yet?
+ self.redirect(self.authorize_redirect("read_stream"))
+ return
+ self.render("stream.html", stream=stream)
+
+ """
+ self.require_setting("facebook_api_key", "Facebook Connect")
+ self.require_setting("facebook_secret", "Facebook Connect")
+ if not method.startswith("facebook."):
+ method = "facebook." + method
+ args["api_key"] = self.settings["facebook_api_key"]
+ args["v"] = "1.0"
+ args["method"] = method
+ args["call_id"] = str(long(time.time() * 1e6))
+ args["format"] = "json"
+ args["sig"] = self._signature(args)
+ url = "http://api.facebook.com/restserver.php?" + \
+ urllib.urlencode(args)
+ http = httpclient.AsyncHTTPClient()
+ http.fetch(url, callback=self.async_callback(
+ self._parse_response, callback))
+
+ def _on_get_user_info(self, callback, session, users):
+ if users is None:
+ callback(None)
+ return
+ callback({
+ "name": users[0]["name"],
+ "first_name": users[0]["first_name"],
+ "last_name": users[0]["last_name"],
+ "uid": users[0]["uid"],
+ "locale": users[0]["locale"],
+ "pic_square": users[0]["pic_square"],
+ "profile_url": users[0]["profile_url"],
+ "username": users[0].get("username"),
+ "session_key": session["session_key"],
+ "session_expires": session.get("expires"),
+ })
+
+ def _parse_response(self, callback, response):
+ if response.error:
+ logging.warning("HTTP error from Facebook: %s", response.error)
+ callback(None)
+ return
+ try:
+ json = escape.json_decode(response.body)
+ except:
+ logging.warning("Invalid JSON from Facebook: %r", response.body)
+ callback(None)
+ return
+ if isinstance(json, dict) and json.get("error_code"):
+ logging.warning("Facebook error: %d: %r", json["error_code"],
+ json.get("error_msg"))
+ callback(None)
+ return
+ callback(json)
+
+ def _signature(self, args):
+ parts = ["%s=%s" % (n, args[n]) for n in sorted(args.keys())]
+ body = "".join(parts) + self.settings["facebook_secret"]
+ if isinstance(body, unicode): body = body.encode("utf-8")
+ return hashlib.md5(body).hexdigest()
+
+
+def _oauth_signature(consumer_token, method, url, parameters={}, token=None):
+ """Calculates the HMAC-SHA1 OAuth signature for the given request.
+
+ See http://oauth.net/core/1.0/#signing_process
+ """
+ parts = urlparse.urlparse(url)
+ scheme, netloc, path = parts[:3]
+ normalized_url = scheme.lower() + "://" + netloc.lower() + path
+
+ base_elems = []
+ base_elems.append(method.upper())
+ base_elems.append(normalized_url)
+ base_elems.append("&".join("%s=%s" % (k, _oauth_escape(str(v)))
+ for k, v in sorted(parameters.items())))
+ base_string = "&".join(_oauth_escape(e) for e in base_elems)
+
+ key_elems = [consumer_token["secret"]]
+ key_elems.append(token["secret"] if token else "")
+ key = "&".join(key_elems)
+
+ hash = hmac.new(key, base_string, hashlib.sha1)
+ return binascii.b2a_base64(hash.digest())[:-1]
+
+
+def _oauth_escape(val):
+ if isinstance(val, unicode):
+ val = val.encode("utf-8")
+ return urllib.quote(val, safe="~")
+
+
+def _oauth_parse_response(body):
+ p = cgi.parse_qs(body, keep_blank_values=False)
+ token = dict(key=p["oauth_token"][0], secret=p["oauth_token_secret"][0])
+
+ # Add the extra parameters the Provider included to the token
+ special = ("oauth_token", "oauth_token_secret")
+ token.update((k, p[k][0]) for k in p if k not in special)
+ return token
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""A module to automatically restart the server when a module is modified.
+
+This module depends on IOLoop, so it will not work in WSGI applications
+and Google AppEngine.
+"""
+
+import functools
+import ioloop
+import logging
+import os
+import sys
+import types
+
+try:
+ import signal
+except ImportError:
+ signal = None
+
+def start(io_loop=None, check_time=500):
+ """Restarts the process automatically when a module is modified.
+
+ We run on the I/O loop, and restarting is a destructive operation,
+ so will terminate any pending requests.
+ """
+ io_loop = io_loop or ioloop.IOLoop.instance()
+ modify_times = {}
+ callback = functools.partial(_reload_on_update, io_loop, modify_times)
+ scheduler = ioloop.PeriodicCallback(callback, check_time, io_loop=io_loop)
+ scheduler.start()
+
+
+_reload_attempted = False
+
+def _reload_on_update(io_loop, modify_times):
+ global _reload_attempted
+ if _reload_attempted:
+ # We already tried to reload and it didn't work, so don't try again.
+ return
+ for module in sys.modules.values():
+ # Some modules play games with sys.modules (e.g. email/__init__.py
+ # in the standard library), and occasionally this can cause strange
+ # failures in getattr. Just ignore anything that's not an ordinary
+ # module.
+ if not isinstance(module, types.ModuleType): continue
+ path = getattr(module, "__file__", None)
+ if not path: continue
+ if path.endswith(".pyc") or path.endswith(".pyo"):
+ path = path[:-1]
+ try:
+ modified = os.stat(path).st_mtime
+ except:
+ continue
+ if path not in modify_times:
+ modify_times[path] = modified
+ continue
+ if modify_times[path] != modified:
+ logging.info("%s modified; restarting server", path)
+ _reload_attempted = True
+ for fd in io_loop._handlers.keys():
+ try:
+ os.close(fd)
+ except:
+ pass
+ if hasattr(signal, "setitimer"):
+ # Clear the alarm signal set by
+ # ioloop.set_blocking_log_threshold so it doesn't fire
+ # after the exec.
+ signal.setitimer(signal.ITIMER_REAL, 0, 0)
+ try:
+ os.execv(sys.executable, [sys.executable] + sys.argv)
+ except OSError:
+ # Mac OS X versions prior to 10.6 do not support execv in
+ # a process that contains multiple threads. Instead of
+ # re-executing in the current process, start a new one
+ # and cause the current process to exit. This isn't
+ # ideal since the new process is detached from the parent
+ # terminal and thus cannot easily be killed with ctrl-C,
+ # but it's better than not being able to autoreload at
+ # all.
+ # Unfortunately the errno returned in this case does not
+ # appear to be consistent, so we can't easily check for
+ # this error specifically.
+ os.spawnv(os.P_NOWAIT, sys.executable,
+ [sys.executable] + sys.argv)
+ sys.exit(0)
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""A lightweight wrapper around MySQLdb."""
+
+import copy
+import MySQLdb.constants
+import MySQLdb.converters
+import MySQLdb.cursors
+import itertools
+import logging
+
+class Connection(object):
+ """A lightweight wrapper around MySQLdb DB-API connections.
+
+ The main value we provide is wrapping rows in a dict/object so that
+ columns can be accessed by name. Typical usage:
+
+ db = database.Connection("localhost", "mydatabase")
+ for article in db.query("SELECT * FROM articles"):
+ print article.title
+
+ Cursors are hidden by the implementation, but other than that, the methods
+ are very similar to the DB-API.
+
+ We explicitly set the timezone to UTC and the character encoding to
+ UTF-8 on all connections to avoid time zone and encoding errors.
+ """
+ def __init__(self, host, database, user=None, password=None):
+ self.host = host
+ self.database = database
+
+ args = dict(conv=CONVERSIONS, use_unicode=True, charset="utf8",
+ db=database, init_command='SET time_zone = "+0:00"',
+ sql_mode="TRADITIONAL")
+ if user is not None:
+ args["user"] = user
+ if password is not None:
+ args["passwd"] = password
+
+ # We accept a path to a MySQL socket file or a host(:port) string
+ if "/" in host:
+ args["unix_socket"] = host
+ else:
+ self.socket = None
+ pair = host.split(":")
+ if len(pair) == 2:
+ args["host"] = pair[0]
+ args["port"] = int(pair[1])
+ else:
+ args["host"] = host
+ args["port"] = 3306
+
+ self._db = None
+ self._db_args = args
+ try:
+ self.reconnect()
+ except:
+ logging.error("Cannot connect to MySQL on %s", self.host,
+ exc_info=True)
+
+ def __del__(self):
+ self.close()
+
+ def close(self):
+ """Closes this database connection."""
+ if getattr(self, "_db", None) is not None:
+ self._db.close()
+ self._db = None
+
+ def reconnect(self):
+ """Closes the existing database connection and re-opens it."""
+ self.close()
+ self._db = MySQLdb.connect(**self._db_args)
+ self._db.autocommit(True)
+
+ def iter(self, query, *parameters):
+ """Returns an iterator for the given query and parameters."""
+ if self._db is None: self.reconnect()
+ cursor = MySQLdb.cursors.SSCursor(self._db)
+ try:
+ self._execute(cursor, query, parameters)
+ column_names = [d[0] for d in cursor.description]
+ for row in cursor:
+ yield Row(zip(column_names, row))
+ finally:
+ cursor.close()
+
+ def query(self, query, *parameters):
+ """Returns a row list for the given query and parameters."""
+ cursor = self._cursor()
+ try:
+ self._execute(cursor, query, parameters)
+ column_names = [d[0] for d in cursor.description]
+ return [Row(itertools.izip(column_names, row)) for row in cursor]
+ finally:
+ cursor.close()
+
+ def get(self, query, *parameters):
+ """Returns the first row returned for the given query."""
+ rows = self.query(query, *parameters)
+ if not rows:
+ return None
+ elif len(rows) > 1:
+ raise Exception("Multiple rows returned for Database.get() query")
+ else:
+ return rows[0]
+
+ def execute(self, query, *parameters):
+ """Executes the given query, returning the lastrowid from the query."""
+ cursor = self._cursor()
+ try:
+ self._execute(cursor, query, parameters)
+ return cursor.lastrowid
+ finally:
+ cursor.close()
+
+ def executemany(self, query, parameters):
+ """Executes the given query against all the given param sequences.
+
+ We return the lastrowid from the query.
+ """
+ cursor = self._cursor()
+ try:
+ cursor.executemany(query, parameters)
+ return cursor.lastrowid
+ finally:
+ cursor.close()
+
+ def _cursor(self):
+ if self._db is None: self.reconnect()
+ return self._db.cursor()
+
+ def _execute(self, cursor, query, parameters):
+ try:
+ return cursor.execute(query, parameters)
+ except OperationalError:
+ logging.error("Error connecting to MySQL on %s", self.host)
+ self.close()
+ raise
+
+
+class Row(dict):
+ """A dict that allows for object-like property access syntax."""
+ def __getattr__(self, name):
+ try:
+ return self[name]
+ except KeyError:
+ raise AttributeError(name)
+
+
+# Fix the access conversions to properly recognize unicode/binary
+FIELD_TYPE = MySQLdb.constants.FIELD_TYPE
+FLAG = MySQLdb.constants.FLAG
+CONVERSIONS = copy.deepcopy(MySQLdb.converters.conversions)
+
+field_types = [FIELD_TYPE.BLOB, FIELD_TYPE.STRING, FIELD_TYPE.VAR_STRING]
+if 'VARCHAR' in vars(FIELD_TYPE):
+ field_types.append(FIELD_TYPE.VARCHAR)
+
+for field_type in field_types:
+ CONVERSIONS[field_type].insert(0, (FLAG.BINARY, str))
+
+
+# Alias some common MySQL exceptions
+IntegrityError = MySQLdb.IntegrityError
+OperationalError = MySQLdb.OperationalError
--- /dev/null
+/*
+ * Copyright 2009 Facebook
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "Python.h"
+#include <string.h>
+#include <sys/epoll.h>
+
+#define MAX_EVENTS 24
+
+/*
+ * Simple wrapper around epoll_create.
+ */
+static PyObject* _epoll_create(void) {
+ int fd = epoll_create(MAX_EVENTS);
+ if (fd == -1) {
+ PyErr_SetFromErrno(PyExc_Exception);
+ return NULL;
+ }
+
+ return PyInt_FromLong(fd);
+}
+
+/*
+ * Simple wrapper around epoll_ctl. We throw an exception if the call fails
+ * rather than returning the error code since it is an infrequent (and likely
+ * catastrophic) event when it does happen.
+ */
+static PyObject* _epoll_ctl(PyObject* self, PyObject* args) {
+ int epfd, op, fd, events;
+ struct epoll_event event;
+
+ if (!PyArg_ParseTuple(args, "iiiI", &epfd, &op, &fd, &events)) {
+ return NULL;
+ }
+
+ memset(&event, 0, sizeof(event));
+ event.events = events;
+ event.data.fd = fd;
+ if (epoll_ctl(epfd, op, fd, &event) == -1) {
+ PyErr_SetFromErrno(PyExc_OSError);
+ return NULL;
+ }
+
+ Py_INCREF(Py_None);
+ return Py_None;
+}
+
+/*
+ * Simple wrapper around epoll_wait. We return None if the call times out and
+ * throw an exception if an error occurs. Otherwise, we return a list of
+ * (fd, event) tuples.
+ */
+static PyObject* _epoll_wait(PyObject* self, PyObject* args) {
+ struct epoll_event events[MAX_EVENTS];
+ int epfd, timeout, num_events, i;
+ PyObject* list;
+ PyObject* tuple;
+
+ if (!PyArg_ParseTuple(args, "ii", &epfd, &timeout)) {
+ return NULL;
+ }
+
+ Py_BEGIN_ALLOW_THREADS
+ num_events = epoll_wait(epfd, events, MAX_EVENTS, timeout);
+ Py_END_ALLOW_THREADS
+ if (num_events == -1) {
+ PyErr_SetFromErrno(PyExc_Exception);
+ return NULL;
+ }
+
+ list = PyList_New(num_events);
+ for (i = 0; i < num_events; i++) {
+ tuple = PyTuple_New(2);
+ PyTuple_SET_ITEM(tuple, 0, PyInt_FromLong(events[i].data.fd));
+ PyTuple_SET_ITEM(tuple, 1, PyInt_FromLong(events[i].events));
+ PyList_SET_ITEM(list, i, tuple);
+ }
+ return list;
+}
+
+/*
+ * Our method declararations
+ */
+static PyMethodDef kEpollMethods[] = {
+ {"epoll_create", (PyCFunction)_epoll_create, METH_NOARGS,
+ "Create an epoll file descriptor"},
+ {"epoll_ctl", _epoll_ctl, METH_VARARGS,
+ "Control an epoll file descriptor"},
+ {"epoll_wait", _epoll_wait, METH_VARARGS,
+ "Wait for events on an epoll file descriptor"},
+ {NULL, NULL, 0, NULL}
+};
+
+/*
+ * Module initialization
+ */
+PyMODINIT_FUNC initepoll(void) {
+ Py_InitModule("epoll", kEpollMethods);
+}
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Escaping/unescaping methods for HTML, JSON, URLs, and others."""
+
+import htmlentitydefs
+import re
+import xml.sax.saxutils
+import urllib
+
+try:
+ import json
+ assert hasattr(json, "loads") and hasattr(json, "dumps")
+ _json_decode = lambda s: json.loads(s)
+ _json_encode = lambda v: json.dumps(v)
+except:
+ try:
+ import simplejson
+ _json_decode = lambda s: simplejson.loads(_unicode(s))
+ _json_encode = lambda v: simplejson.dumps(v)
+ except ImportError:
+ try:
+ # For Google AppEngine
+ from django.utils import simplejson
+ _json_decode = lambda s: simplejson.loads(_unicode(s))
+ _json_encode = lambda v: simplejson.dumps(v)
+ except ImportError:
+ raise Exception("A JSON parser is required, e.g., simplejson at "
+ "http://pypi.python.org/pypi/simplejson/")
+
+
+def xhtml_escape(value):
+ """Escapes a string so it is valid within XML or XHTML."""
+ return utf8(xml.sax.saxutils.escape(value, {'"': """}))
+
+
+def xhtml_unescape(value):
+ """Un-escapes an XML-escaped string."""
+ return re.sub(r"&(#?)(\w+?);", _convert_entity, _unicode(value))
+
+
+def json_encode(value):
+ """JSON-encodes the given Python object."""
+ # JSON permits but does not require forward slashes to be escaped.
+ # This is useful when json data is emitted in a <script> tag
+ # in HTML, as it prevents </script> tags from prematurely terminating
+ # the javscript. Some json libraries do this escaping by default,
+ # although python's standard library does not, so we do it here.
+ # http://stackoverflow.com/questions/1580647/json-why-are-forward-slashes-escaped
+ return _json_encode(value).replace("</", "<\\/")
+
+
+def json_decode(value):
+ """Returns Python objects for the given JSON string."""
+ return _json_decode(value)
+
+
+def squeeze(value):
+ """Replace all sequences of whitespace chars with a single space."""
+ return re.sub(r"[\x00-\x20]+", " ", value).strip()
+
+
+def url_escape(value):
+ """Returns a valid URL-encoded version of the given value."""
+ return urllib.quote_plus(utf8(value))
+
+
+def url_unescape(value):
+ """Decodes the given value from a URL."""
+ return _unicode(urllib.unquote_plus(value))
+
+
+def utf8(value):
+ if isinstance(value, unicode):
+ return value.encode("utf-8")
+ assert isinstance(value, str)
+ return value
+
+
+def _unicode(value):
+ if isinstance(value, str):
+ return value.decode("utf-8")
+ assert isinstance(value, unicode)
+ return value
+
+
+def _convert_entity(m):
+ if m.group(1) == "#":
+ try:
+ return unichr(int(m.group(2)))
+ except ValueError:
+ return "&#%s;" % m.group(2)
+ try:
+ return _HTML_UNICODE_MAP[m.group(2)]
+ except KeyError:
+ return "&%s;" % m.group(2)
+
+
+def _build_unicode_map():
+ unicode_map = {}
+ for name, value in htmlentitydefs.name2codepoint.iteritems():
+ unicode_map[name] = unichr(value)
+ return unicode_map
+
+_HTML_UNICODE_MAP = _build_unicode_map()
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Blocking and non-blocking HTTP client implementations using pycurl."""
+
+import calendar
+import collections
+import cStringIO
+import email.utils
+import errno
+import escape
+import httplib
+import httputil
+import ioloop
+import logging
+import pycurl
+import sys
+import time
+import weakref
+
+class HTTPClient(object):
+ """A blocking HTTP client backed with pycurl.
+
+ Typical usage looks like this:
+
+ http_client = httpclient.HTTPClient()
+ try:
+ response = http_client.fetch("http://www.google.com/")
+ print response.body
+ except httpclient.HTTPError, e:
+ print "Error:", e
+
+ fetch() can take a string URL or an HTTPRequest instance, which offers
+ more options, like executing POST/PUT/DELETE requests.
+ """
+ def __init__(self, max_simultaneous_connections=None):
+ self._curl = _curl_create(max_simultaneous_connections)
+
+ def __del__(self):
+ self._curl.close()
+
+ def fetch(self, request, **kwargs):
+ """Executes an HTTPRequest, returning an HTTPResponse.
+
+ If an error occurs during the fetch, we raise an HTTPError.
+ """
+ if not isinstance(request, HTTPRequest):
+ request = HTTPRequest(url=request, **kwargs)
+ buffer = cStringIO.StringIO()
+ headers = httputil.HTTPHeaders()
+ try:
+ _curl_setup_request(self._curl, request, buffer, headers)
+ self._curl.perform()
+ code = self._curl.getinfo(pycurl.HTTP_CODE)
+ effective_url = self._curl.getinfo(pycurl.EFFECTIVE_URL)
+ buffer.seek(0)
+ response = HTTPResponse(
+ request=request, code=code, headers=headers,
+ buffer=buffer, effective_url=effective_url)
+ if code < 200 or code >= 300:
+ raise HTTPError(code, response=response)
+ return response
+ except pycurl.error, e:
+ buffer.close()
+ raise CurlError(*e)
+
+
+class AsyncHTTPClient(object):
+ """An non-blocking HTTP client backed with pycurl.
+
+ Example usage:
+
+ import ioloop
+
+ def handle_request(response):
+ if response.error:
+ print "Error:", response.error
+ else:
+ print response.body
+ ioloop.IOLoop.instance().stop()
+
+ http_client = httpclient.AsyncHTTPClient()
+ http_client.fetch("http://www.google.com/", handle_request)
+ ioloop.IOLoop.instance().start()
+
+ fetch() can take a string URL or an HTTPRequest instance, which offers
+ more options, like executing POST/PUT/DELETE requests.
+
+ The keyword argument max_clients to the AsyncHTTPClient constructor
+ determines the maximum number of simultaneous fetch() operations that
+ can execute in parallel on each IOLoop.
+ """
+ _ASYNC_CLIENTS = weakref.WeakKeyDictionary()
+
+ def __new__(cls, io_loop=None, max_clients=10,
+ max_simultaneous_connections=None):
+ # There is one client per IOLoop since they share curl instances
+ io_loop = io_loop or ioloop.IOLoop.instance()
+ if io_loop in cls._ASYNC_CLIENTS:
+ return cls._ASYNC_CLIENTS[io_loop]
+ else:
+ instance = super(AsyncHTTPClient, cls).__new__(cls)
+ instance.io_loop = io_loop
+ instance._multi = pycurl.CurlMulti()
+ instance._curls = [_curl_create(max_simultaneous_connections)
+ for i in xrange(max_clients)]
+ instance._free_list = instance._curls[:]
+ instance._requests = collections.deque()
+ instance._fds = {}
+ instance._events = {}
+ instance._added_perform_callback = False
+ instance._timeout = None
+ instance._closed = False
+ cls._ASYNC_CLIENTS[io_loop] = instance
+ return instance
+
+ def close(self):
+ """Destroys this http client, freeing any file descriptors used.
+ Not needed in normal use, but may be helpful in unittests that
+ create and destroy http clients. No other methods may be called
+ on the AsyncHTTPClient after close().
+ """
+ del AsyncHTTPClient._ASYNC_CLIENTS[self.io_loop]
+ for curl in self._curls:
+ curl.close()
+ self._multi.close()
+ self._closed = True
+
+ def fetch(self, request, callback, **kwargs):
+ """Executes an HTTPRequest, calling callback with an HTTPResponse.
+
+ If an error occurs during the fetch, the HTTPResponse given to the
+ callback has a non-None error attribute that contains the exception
+ encountered during the request. You can call response.reraise() to
+ throw the exception (if any) in the callback.
+ """
+ if not isinstance(request, HTTPRequest):
+ request = HTTPRequest(url=request, **kwargs)
+ self._requests.append((request, callback))
+ self._add_perform_callback()
+
+ def _add_perform_callback(self):
+ if not self._added_perform_callback:
+ self.io_loop.add_callback(self._perform)
+ self._added_perform_callback = True
+
+ def _handle_events(self, fd, events):
+ self._events[fd] = events
+ self._add_perform_callback()
+
+ def _handle_timeout(self):
+ self._timeout = None
+ self._perform()
+
+ def _perform(self):
+ self._added_perform_callback = False
+
+ if self._closed:
+ return
+
+ while True:
+ while True:
+ ret, num_handles = self._multi.perform()
+ if ret != pycurl.E_CALL_MULTI_PERFORM:
+ break
+
+ # Update the set of active file descriptors. It is important
+ # that this happen immediately after perform() because
+ # fds that have been removed from fdset are free to be reused
+ # in user callbacks.
+ fds = {}
+ (readable, writable, exceptable) = self._multi.fdset()
+ for fd in readable:
+ fds[fd] = fds.get(fd, 0) | 0x1 | 0x2
+ for fd in writable:
+ fds[fd] = fds.get(fd, 0) | 0x4
+ for fd in exceptable:
+ fds[fd] = fds.get(fd, 0) | 0x8 | 0x10
+
+ if fds and max(fds.iterkeys()) > 900:
+ # Libcurl has a bug in which it behaves unpredictably with
+ # file descriptors greater than 1024. (This is because
+ # even though it uses poll() instead of select(), it still
+ # uses FD_SET internally) Since curl opens its own file
+ # descriptors we can't catch this problem when it happens,
+ # and the best we can do is detect that it's about to
+ # happen. Exiting is a lousy way to handle this error,
+ # but there's not much we can do at this point. Exiting
+ # (and getting restarted by whatever monitoring process
+ # is handling crashed tornado processes) will at least
+ # get things working again and hopefully bring the issue
+ # to someone's attention.
+ # If you run into this issue, you either have a file descriptor
+ # leak or need to run more tornado processes (so that none
+ # of them are handling more than 1000 simultaneous connections)
+ print >> sys.stderr, "ERROR: File descriptor too high for libcurl. Exiting."
+ logging.error("File descriptor too high for libcurl. Exiting.")
+ sys.exit(1)
+
+ for fd in self._fds:
+ if fd not in fds:
+ try:
+ self.io_loop.remove_handler(fd)
+ except (OSError, IOError), e:
+ if e[0] != errno.ENOENT:
+ raise
+
+ for fd, events in fds.iteritems():
+ old_events = self._fds.get(fd, None)
+ if old_events is None:
+ self.io_loop.add_handler(fd, self._handle_events, events)
+ elif old_events != events:
+ try:
+ self.io_loop.update_handler(fd, events)
+ except (OSError, IOError), e:
+ if e[0] == errno.ENOENT:
+ self.io_loop.add_handler(fd, self._handle_events,
+ events)
+ else:
+ raise
+ self._fds = fds
+
+
+ # Handle completed fetches
+ completed = 0
+ while True:
+ num_q, ok_list, err_list = self._multi.info_read()
+ for curl in ok_list:
+ self._finish(curl)
+ completed += 1
+ for curl, errnum, errmsg in err_list:
+ self._finish(curl, errnum, errmsg)
+ completed += 1
+ if num_q == 0:
+ break
+
+ # Start fetching new URLs
+ started = 0
+ while self._free_list and self._requests:
+ started += 1
+ curl = self._free_list.pop()
+ (request, callback) = self._requests.popleft()
+ curl.info = {
+ "headers": httputil.HTTPHeaders(),
+ "buffer": cStringIO.StringIO(),
+ "request": request,
+ "callback": callback,
+ "start_time": time.time(),
+ }
+ _curl_setup_request(curl, request, curl.info["buffer"],
+ curl.info["headers"])
+ self._multi.add_handle(curl)
+
+ if not started and not completed:
+ break
+
+ if self._timeout is not None:
+ self.io_loop.remove_timeout(self._timeout)
+ self._timeout = None
+
+ if num_handles:
+ self._timeout = self.io_loop.add_timeout(
+ time.time() + 0.2, self._handle_timeout)
+
+
+ def _finish(self, curl, curl_error=None, curl_message=None):
+ info = curl.info
+ curl.info = None
+ self._multi.remove_handle(curl)
+ self._free_list.append(curl)
+ buffer = info["buffer"]
+ if curl_error:
+ error = CurlError(curl_error, curl_message)
+ code = error.code
+ body = None
+ effective_url = None
+ buffer.close()
+ buffer = None
+ else:
+ error = None
+ code = curl.getinfo(pycurl.HTTP_CODE)
+ effective_url = curl.getinfo(pycurl.EFFECTIVE_URL)
+ buffer.seek(0)
+ try:
+ info["callback"](HTTPResponse(
+ request=info["request"], code=code, headers=info["headers"],
+ buffer=buffer, effective_url=effective_url, error=error,
+ request_time=time.time() - info["start_time"]))
+ except (KeyboardInterrupt, SystemExit):
+ raise
+ except:
+ logging.error("Exception in callback %r", info["callback"],
+ exc_info=True)
+
+
+class AsyncHTTPClient2(object):
+ """Alternate implementation of AsyncHTTPClient.
+
+ This class has the same interface as AsyncHTTPClient (so see that class
+ for usage documentation) but is implemented with a different set of
+ libcurl APIs (curl_multi_socket_action instead of fdset/perform).
+ This implementation will likely become the default in the future, but
+ for now should be considered somewhat experimental.
+
+ The main advantage of this class over the original implementation is
+ that it is immune to the fd > 1024 bug, so applications with a large
+ number of simultaneous requests (e.g. long-polling) may prefer this
+ version.
+
+ Known bugs:
+ * Timeouts connecting to localhost
+ In some situations, this implementation will return a connection
+ timeout when the old implementation would be able to connect. This
+ has only been observed when connecting to localhost when using
+ the kqueue-based IOLoop (mac/bsd), but it may also occur on epoll (linux)
+ and, in principle, for non-localhost sites.
+ While the bug is unrelated to IPv6, disabling IPv6 will avoid the
+ most common manifestations of the bug, so this class disables IPv6 when
+ it detects an affected version of libcurl.
+ The underlying cause is a libcurl bug in versions up to and including
+ 7.21.0 (it will be fixed in the not-yet-released 7.21.1)
+ http://sourceforge.net/tracker/?func=detail&aid=3017819&group_id=976&atid=100976
+ """
+ _ASYNC_CLIENTS = weakref.WeakKeyDictionary()
+
+ def __new__(cls, io_loop=None, max_clients=10,
+ max_simultaneous_connections=None):
+ # There is one client per IOLoop since they share curl instances
+ io_loop = io_loop or ioloop.IOLoop.instance()
+ if io_loop in cls._ASYNC_CLIENTS:
+ return cls._ASYNC_CLIENTS[io_loop]
+ else:
+ instance = super(AsyncHTTPClient2, cls).__new__(cls)
+ instance.io_loop = io_loop
+ instance._multi = pycurl.CurlMulti()
+ instance._multi.setopt(pycurl.M_TIMERFUNCTION,
+ instance._set_timeout)
+ instance._multi.setopt(pycurl.M_SOCKETFUNCTION,
+ instance._handle_socket)
+ instance._curls = [_curl_create(max_simultaneous_connections)
+ for i in xrange(max_clients)]
+ instance._free_list = instance._curls[:]
+ instance._requests = collections.deque()
+ instance._fds = {}
+ instance._timeout = None
+ cls._ASYNC_CLIENTS[io_loop] = instance
+ return instance
+
+ def close(self):
+ """Destroys this http client, freeing any file descriptors used.
+ Not needed in normal use, but may be helpful in unittests that
+ create and destroy http clients. No other methods may be called
+ on the AsyncHTTPClient after close().
+ """
+ del AsyncHTTPClient2._ASYNC_CLIENTS[self.io_loop]
+ for curl in self._curls:
+ curl.close()
+ self._multi.close()
+ self._closed = True
+
+ def fetch(self, request, callback, **kwargs):
+ """Executes an HTTPRequest, calling callback with an HTTPResponse.
+
+ If an error occurs during the fetch, the HTTPResponse given to the
+ callback has a non-None error attribute that contains the exception
+ encountered during the request. You can call response.reraise() to
+ throw the exception (if any) in the callback.
+ """
+ if not isinstance(request, HTTPRequest):
+ request = HTTPRequest(url=request, **kwargs)
+ self._requests.append((request, callback))
+ self._process_queue()
+ self._set_timeout(0)
+
+ def _handle_socket(self, event, fd, multi, data):
+ """Called by libcurl when it wants to change the file descriptors
+ it cares about.
+ """
+ event_map = {
+ pycurl.POLL_NONE: ioloop.IOLoop.NONE,
+ pycurl.POLL_IN: ioloop.IOLoop.READ,
+ pycurl.POLL_OUT: ioloop.IOLoop.WRITE,
+ pycurl.POLL_INOUT: ioloop.IOLoop.READ | ioloop.IOLoop.WRITE
+ }
+ if event == pycurl.POLL_REMOVE:
+ self.io_loop.remove_handler(fd)
+ del self._fds[fd]
+ else:
+ ioloop_event = event_map[event]
+ if fd not in self._fds:
+ self._fds[fd] = ioloop_event
+ self.io_loop.add_handler(fd, self._handle_events,
+ ioloop_event)
+ else:
+ self._fds[fd] = ioloop_event
+ self.io_loop.update_handler(fd, ioloop_event)
+
+ def _set_timeout(self, msecs):
+ """Called by libcurl to schedule a timeout."""
+ if self._timeout is not None:
+ self.io_loop.remove_timeout(self._timeout)
+ self._timeout = self.io_loop.add_timeout(
+ time.time() + msecs/1000.0, self._handle_timeout)
+
+ def _handle_events(self, fd, events):
+ """Called by IOLoop when there is activity on one of our
+ file descriptors.
+ """
+ action = 0
+ if events & ioloop.IOLoop.READ: action |= pycurl.CSELECT_IN
+ if events & ioloop.IOLoop.WRITE: action |= pycurl.CSELECT_OUT
+ while True:
+ try:
+ ret, num_handles = self._multi.socket_action(fd, action)
+ except Exception, e:
+ ret = e[0]
+ if ret != pycurl.E_CALL_MULTI_PERFORM:
+ break
+ self._finish_pending_requests()
+
+ def _handle_timeout(self):
+ """Called by IOLoop when the requested timeout has passed."""
+ self._timeout = None
+ while True:
+ try:
+ ret, num_handles = self._multi.socket_action(
+ pycurl.SOCKET_TIMEOUT, 0)
+ except Exception, e:
+ ret = e[0]
+ if ret != pycurl.E_CALL_MULTI_PERFORM:
+ break
+ self._finish_pending_requests()
+
+ # In theory, we shouldn't have to do this because curl will
+ # call _set_timeout whenever the timeout changes. However,
+ # sometimes after _handle_timeout we will need to reschedule
+ # immediately even though nothing has changed from curl's
+ # perspective. This is because when socket_action is
+ # called with SOCKET_TIMEOUT, libcurl decides internally which
+ # timeouts need to be processed by using a monotonic clock
+ # (where available) while tornado uses python's time.time()
+ # to decide when timeouts have occurred. When those clocks
+ # disagree on elapsed time (as they will whenever there is an
+ # NTP adjustment), tornado might call _handle_timeout before
+ # libcurl is ready. After each timeout, resync the scheduled
+ # timeout with libcurl's current state.
+ new_timeout = self._multi.timeout()
+ if new_timeout != -1:
+ self._set_timeout(new_timeout)
+
+ def _finish_pending_requests(self):
+ """Process any requests that were completed by the last
+ call to multi.socket_action.
+ """
+ while True:
+ num_q, ok_list, err_list = self._multi.info_read()
+ for curl in ok_list:
+ self._finish(curl)
+ for curl, errnum, errmsg in err_list:
+ self._finish(curl, errnum, errmsg)
+ if num_q == 0:
+ break
+ self._process_queue()
+
+ def _process_queue(self):
+ while True:
+ started = 0
+ while self._free_list and self._requests:
+ started += 1
+ curl = self._free_list.pop()
+ (request, callback) = self._requests.popleft()
+ curl.info = {
+ "headers": httputil.HTTPHeaders(),
+ "buffer": cStringIO.StringIO(),
+ "request": request,
+ "callback": callback,
+ "start_time": time.time(),
+ }
+ # Disable IPv6 to mitigate the effects of this bug
+ # on curl versions <= 7.21.0
+ # http://sourceforge.net/tracker/?func=detail&aid=3017819&group_id=976&atid=100976
+ if pycurl.version_info()[2] <= 0x71500: # 7.21.0
+ curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
+ _curl_setup_request(curl, request, curl.info["buffer"],
+ curl.info["headers"])
+ self._multi.add_handle(curl)
+
+ if not started:
+ break
+
+ def _finish(self, curl, curl_error=None, curl_message=None):
+ info = curl.info
+ curl.info = None
+ self._multi.remove_handle(curl)
+ self._free_list.append(curl)
+ buffer = info["buffer"]
+ if curl_error:
+ error = CurlError(curl_error, curl_message)
+ code = error.code
+ effective_url = None
+ buffer.close()
+ buffer = None
+ else:
+ error = None
+ code = curl.getinfo(pycurl.HTTP_CODE)
+ effective_url = curl.getinfo(pycurl.EFFECTIVE_URL)
+ buffer.seek(0)
+ try:
+ info["callback"](HTTPResponse(
+ request=info["request"], code=code, headers=info["headers"],
+ buffer=buffer, effective_url=effective_url, error=error,
+ request_time=time.time() - info["start_time"]))
+ except (KeyboardInterrupt, SystemExit):
+ raise
+ except:
+ logging.error("Exception in callback %r", info["callback"],
+ exc_info=True)
+
+
+class HTTPRequest(object):
+ def __init__(self, url, method="GET", headers=None, body=None,
+ auth_username=None, auth_password=None,
+ connect_timeout=20.0, request_timeout=20.0,
+ if_modified_since=None, follow_redirects=True,
+ max_redirects=5, user_agent=None, use_gzip=True,
+ network_interface=None, streaming_callback=None,
+ header_callback=None, prepare_curl_callback=None,
+ allow_nonstandard_methods=False):
+ if headers is None:
+ headers = httputil.HTTPHeaders()
+ if if_modified_since:
+ timestamp = calendar.timegm(if_modified_since.utctimetuple())
+ headers["If-Modified-Since"] = email.utils.formatdate(
+ timestamp, localtime=False, usegmt=True)
+ if "Pragma" not in headers:
+ headers["Pragma"] = ""
+ self.url = _utf8(url)
+ self.method = method
+ self.headers = headers
+ self.body = body
+ self.auth_username = _utf8(auth_username)
+ self.auth_password = _utf8(auth_password)
+ self.connect_timeout = connect_timeout
+ self.request_timeout = request_timeout
+ self.follow_redirects = follow_redirects
+ self.max_redirects = max_redirects
+ self.user_agent = user_agent
+ self.use_gzip = use_gzip
+ self.network_interface = network_interface
+ self.streaming_callback = streaming_callback
+ self.header_callback = header_callback
+ self.prepare_curl_callback = prepare_curl_callback
+ self.allow_nonstandard_methods = allow_nonstandard_methods
+
+
+class HTTPResponse(object):
+ def __init__(self, request, code, headers={}, buffer=None, effective_url=None,
+ error=None, request_time=None):
+ self.request = request
+ self.code = code
+ self.headers = headers
+ self.buffer = buffer
+ self._body = None
+ if effective_url is None:
+ self.effective_url = request.url
+ else:
+ self.effective_url = effective_url
+ if error is None:
+ if self.code < 200 or self.code >= 300:
+ self.error = HTTPError(self.code, response=self)
+ else:
+ self.error = None
+ else:
+ self.error = error
+ self.request_time = request_time
+
+ def _get_body(self):
+ if self.buffer is None:
+ return None
+ elif self._body is None:
+ self._body = self.buffer.getvalue()
+
+ return self._body
+
+ body = property(_get_body)
+
+ def rethrow(self):
+ if self.error:
+ raise self.error
+
+ def __repr__(self):
+ args = ",".join("%s=%r" % i for i in self.__dict__.iteritems())
+ return "%s(%s)" % (self.__class__.__name__, args)
+
+ def __del__(self):
+ if self.buffer is not None:
+ self.buffer.close()
+
+
+class HTTPError(Exception):
+ """Exception thrown for an unsuccessful HTTP request.
+
+ Attributes:
+ code - HTTP error integer error code, e.g. 404. Error code 599 is
+ used when no HTTP response was received, e.g. for a timeout.
+ response - HTTPResponse object, if any.
+
+ Note that if follow_redirects is False, redirects become HTTPErrors,
+ and you can look at error.response.headers['Location'] to see the
+ destination of the redirect.
+ """
+ def __init__(self, code, message=None, response=None):
+ self.code = code
+ message = message or httplib.responses.get(code, "Unknown")
+ self.response = response
+ Exception.__init__(self, "HTTP %d: %s" % (self.code, message))
+
+
+class CurlError(HTTPError):
+ def __init__(self, errno, message):
+ HTTPError.__init__(self, 599, message)
+ self.errno = errno
+
+
+def _curl_create(max_simultaneous_connections=None):
+ curl = pycurl.Curl()
+ if logging.getLogger().isEnabledFor(logging.DEBUG):
+ curl.setopt(pycurl.VERBOSE, 1)
+ curl.setopt(pycurl.DEBUGFUNCTION, _curl_debug)
+ curl.setopt(pycurl.MAXCONNECTS, max_simultaneous_connections or 5)
+ return curl
+
+
+def _curl_setup_request(curl, request, buffer, headers):
+ curl.setopt(pycurl.URL, request.url)
+ # Request headers may be either a regular dict or HTTPHeaders object
+ if isinstance(request.headers, httputil.HTTPHeaders):
+ curl.setopt(pycurl.HTTPHEADER,
+ [_utf8("%s: %s" % i) for i in request.headers.get_all()])
+ else:
+ curl.setopt(pycurl.HTTPHEADER,
+ [_utf8("%s: %s" % i) for i in request.headers.iteritems()])
+ if request.header_callback:
+ curl.setopt(pycurl.HEADERFUNCTION, request.header_callback)
+ else:
+ curl.setopt(pycurl.HEADERFUNCTION,
+ lambda line: _curl_header_callback(headers, line))
+ if request.streaming_callback:
+ curl.setopt(pycurl.WRITEFUNCTION, request.streaming_callback)
+ else:
+ curl.setopt(pycurl.WRITEFUNCTION, buffer.write)
+ curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects)
+ curl.setopt(pycurl.MAXREDIRS, request.max_redirects)
+ curl.setopt(pycurl.CONNECTTIMEOUT, int(request.connect_timeout))
+ curl.setopt(pycurl.TIMEOUT, int(request.request_timeout))
+ if request.user_agent:
+ curl.setopt(pycurl.USERAGENT, _utf8(request.user_agent))
+ else:
+ curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)")
+ if request.network_interface:
+ curl.setopt(pycurl.INTERFACE, request.network_interface)
+ if request.use_gzip:
+ curl.setopt(pycurl.ENCODING, "gzip,deflate")
+ else:
+ curl.setopt(pycurl.ENCODING, "none")
+
+ # Set the request method through curl's retarded interface which makes
+ # up names for almost every single method
+ curl_options = {
+ "GET": pycurl.HTTPGET,
+ "POST": pycurl.POST,
+ "PUT": pycurl.UPLOAD,
+ "HEAD": pycurl.NOBODY,
+ }
+ custom_methods = set(["DELETE"])
+ for o in curl_options.values():
+ curl.setopt(o, False)
+ if request.method in curl_options:
+ curl.unsetopt(pycurl.CUSTOMREQUEST)
+ curl.setopt(curl_options[request.method], True)
+ elif request.allow_nonstandard_methods or request.method in custom_methods:
+ curl.setopt(pycurl.CUSTOMREQUEST, request.method)
+ else:
+ raise KeyError('unknown method ' + request.method)
+
+ # Handle curl's cryptic options for every individual HTTP method
+ if request.method in ("POST", "PUT"):
+ request_buffer = cStringIO.StringIO(escape.utf8(request.body))
+ curl.setopt(pycurl.READFUNCTION, request_buffer.read)
+ if request.method == "POST":
+ def ioctl(cmd):
+ if cmd == curl.IOCMD_RESTARTREAD:
+ request_buffer.seek(0)
+ curl.setopt(pycurl.IOCTLFUNCTION, ioctl)
+ curl.setopt(pycurl.POSTFIELDSIZE, len(request.body))
+ else:
+ curl.setopt(pycurl.INFILESIZE, len(request.body))
+
+ if request.auth_username and request.auth_password:
+ userpwd = "%s:%s" % (request.auth_username, request.auth_password)
+ curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
+ curl.setopt(pycurl.USERPWD, userpwd)
+ logging.info("%s %s (username: %r)", request.method, request.url,
+ request.auth_username)
+ else:
+ curl.unsetopt(pycurl.USERPWD)
+ logging.info("%s %s", request.method, request.url)
+ if request.prepare_curl_callback is not None:
+ request.prepare_curl_callback(curl)
+
+
+def _curl_header_callback(headers, header_line):
+ if header_line.startswith("HTTP/"):
+ headers.clear()
+ return
+ if header_line == "\r\n":
+ return
+ headers.parse_line(header_line)
+
+def _curl_debug(debug_type, debug_msg):
+ debug_types = ('I', '<', '>', '<', '>')
+ if debug_type == 0:
+ logging.debug('%s', debug_msg.strip())
+ elif debug_type in (1, 2):
+ for line in debug_msg.splitlines():
+ logging.debug('%s %s', debug_types[debug_type], line)
+ elif debug_type == 4:
+ logging.debug('%s %r', debug_types[debug_type], debug_msg)
+
+
+def _utf8(value):
+ if value is None:
+ return value
+ if isinstance(value, unicode):
+ return value.encode("utf-8")
+ assert isinstance(value, str)
+ return value
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""A non-blocking, single-threaded HTTP server."""
+
+import cgi
+import errno
+import httputil
+import ioloop
+import iostream
+import logging
+import os
+import socket
+import time
+import urlparse
+
+try:
+ import fcntl
+except ImportError:
+ if os.name == 'nt':
+ import win32_support as fcntl
+ else:
+ raise
+
+try:
+ import ssl # Python 2.6+
+except ImportError:
+ ssl = None
+
+class HTTPServer(object):
+ """A non-blocking, single-threaded HTTP server.
+
+ A server is defined by a request callback that takes an HTTPRequest
+ instance as an argument and writes a valid HTTP response with
+ request.write(). request.finish() finishes the request (but does not
+ necessarily close the connection in the case of HTTP/1.1 keep-alive
+ requests). A simple example server that echoes back the URI you
+ requested:
+
+ import httpserver
+ import ioloop
+
+ def handle_request(request):
+ message = "You requested %s\n" % request.uri
+ request.write("HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n%s" % (
+ len(message), message))
+ request.finish()
+
+ http_server = httpserver.HTTPServer(handle_request)
+ http_server.listen(8888)
+ ioloop.IOLoop.instance().start()
+
+ HTTPServer is a very basic connection handler. Beyond parsing the
+ HTTP request body and headers, the only HTTP semantics implemented
+ in HTTPServer is HTTP/1.1 keep-alive connections. We do not, however,
+ implement chunked encoding, so the request callback must provide a
+ Content-Length header or implement chunked encoding for HTTP/1.1
+ requests for the server to run correctly for HTTP/1.1 clients. If
+ the request handler is unable to do this, you can provide the
+ no_keep_alive argument to the HTTPServer constructor, which will
+ ensure the connection is closed on every request no matter what HTTP
+ version the client is using.
+
+ If xheaders is True, we support the X-Real-Ip and X-Scheme headers,
+ which override the remote IP and HTTP scheme for all requests. These
+ headers are useful when running Tornado behind a reverse proxy or
+ load balancer.
+
+ HTTPServer can serve HTTPS (SSL) traffic with Python 2.6+ and OpenSSL.
+ To make this server serve SSL traffic, send the ssl_options dictionary
+ argument with the arguments required for the ssl.wrap_socket() method,
+ including "certfile" and "keyfile":
+
+ HTTPServer(applicaton, ssl_options={
+ "certfile": os.path.join(data_dir, "mydomain.crt"),
+ "keyfile": os.path.join(data_dir, "mydomain.key"),
+ })
+
+ By default, listen() runs in a single thread in a single process. You
+ can utilize all available CPUs on this machine by calling bind() and
+ start() instead of listen():
+
+ http_server = httpserver.HTTPServer(handle_request)
+ http_server.bind(8888)
+ http_server.start() # Forks multiple sub-processes
+ ioloop.IOLoop.instance().start()
+
+ start() detects the number of CPUs on this machine and "pre-forks" that
+ number of child processes so that we have one Tornado process per CPU,
+ all with their own IOLoop. You can also pass in the specific number of
+ child processes you want to run with if you want to override this
+ auto-detection.
+ """
+ def __init__(self, request_callback, no_keep_alive=False, io_loop=None,
+ xheaders=False, ssl_options=None):
+ """Initializes the server with the given request callback.
+
+ If you use pre-forking/start() instead of the listen() method to
+ start your server, you should not pass an IOLoop instance to this
+ constructor. Each pre-forked child process will create its own
+ IOLoop instance after the forking process.
+ """
+ self.request_callback = request_callback
+ self.no_keep_alive = no_keep_alive
+ self.io_loop = io_loop
+ self.xheaders = xheaders
+ self.ssl_options = ssl_options
+ self._socket = None
+ self._started = False
+
+ def listen(self, port, address=""):
+ """Binds to the given port and starts the server in a single process.
+
+ This method is a shortcut for:
+
+ server.bind(port, address)
+ server.start(1)
+
+ """
+ self.bind(port, address)
+ self.start(1)
+
+ def bind(self, port, address=""):
+ """Binds this server to the given port on the given IP address.
+
+ To start the server, call start(). If you want to run this server
+ in a single process, you can call listen() as a shortcut to the
+ sequence of bind() and start() calls.
+ """
+ assert not self._socket
+ self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
+ flags = fcntl.fcntl(self._socket.fileno(), fcntl.F_GETFD)
+ flags |= fcntl.FD_CLOEXEC
+ fcntl.fcntl(self._socket.fileno(), fcntl.F_SETFD, flags)
+ self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ self._socket.setblocking(0)
+ self._socket.bind((address, port))
+ self._socket.listen(128)
+
+ def start(self, num_processes=1):
+ """Starts this server in the IOLoop.
+
+ By default, we run the server in this process and do not fork any
+ additional child process.
+
+ If num_processes is None or <= 0, we detect the number of cores
+ available on this machine and fork that number of child
+ processes. If num_processes is given and > 1, we fork that
+ specific number of sub-processes.
+
+ Since we use processes and not threads, there is no shared memory
+ between any server code.
+ """
+ assert not self._started
+ self._started = True
+ if num_processes is None or num_processes <= 0:
+ # Use sysconf to detect the number of CPUs (cores)
+ try:
+ num_processes = os.sysconf("SC_NPROCESSORS_CONF")
+ except ValueError:
+ logging.error("Could not get num processors from sysconf; "
+ "running with one process")
+ num_processes = 1
+ if num_processes > 1 and ioloop.IOLoop.initialized():
+ logging.error("Cannot run in multiple processes: IOLoop instance "
+ "has already been initialized. You cannot call "
+ "IOLoop.instance() before calling start()")
+ num_processes = 1
+ if num_processes > 1:
+ logging.info("Pre-forking %d server processes", num_processes)
+ for i in range(num_processes):
+ if os.fork() == 0:
+ self.io_loop = ioloop.IOLoop.instance()
+ self.io_loop.add_handler(
+ self._socket.fileno(), self._handle_events,
+ ioloop.IOLoop.READ)
+ return
+ os.waitpid(-1, 0)
+ else:
+ if not self.io_loop:
+ self.io_loop = ioloop.IOLoop.instance()
+ self.io_loop.add_handler(self._socket.fileno(),
+ self._handle_events,
+ ioloop.IOLoop.READ)
+
+ def stop(self):
+ self.io_loop.remove_handler(self._socket.fileno())
+ self._socket.close()
+
+ def _handle_events(self, fd, events):
+ while True:
+ try:
+ connection, address = self._socket.accept()
+ except socket.error, e:
+ if e[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
+ return
+ raise
+ if self.ssl_options is not None:
+ assert ssl, "Python 2.6+ and OpenSSL required for SSL"
+ connection = ssl.wrap_socket(
+ connection, server_side=True, **self.ssl_options)
+ try:
+ stream = iostream.IOStream(connection, io_loop=self.io_loop)
+ HTTPConnection(stream, address, self.request_callback,
+ self.no_keep_alive, self.xheaders)
+ except:
+ logging.error("Error in connection callback", exc_info=True)
+
+
+class HTTPConnection(object):
+ """Handles a connection to an HTTP client, executing HTTP requests.
+
+ We parse HTTP headers and bodies, and execute the request callback
+ until the HTTP conection is closed.
+ """
+ def __init__(self, stream, address, request_callback, no_keep_alive=False,
+ xheaders=False):
+ self.stream = stream
+ self.address = address
+ self.request_callback = request_callback
+ self.no_keep_alive = no_keep_alive
+ self.xheaders = xheaders
+ self._request = None
+ self._request_finished = False
+ self.stream.read_until("\r\n\r\n", self._on_headers)
+
+ def write(self, chunk):
+ assert self._request, "Request closed"
+ if not self.stream.closed():
+ self.stream.write(chunk, self._on_write_complete)
+
+ def finish(self):
+ assert self._request, "Request closed"
+ self._request_finished = True
+ if not self.stream.writing():
+ self._finish_request()
+
+ def _on_write_complete(self):
+ if self._request_finished:
+ self._finish_request()
+
+ def _finish_request(self):
+ if self.no_keep_alive:
+ disconnect = True
+ else:
+ connection_header = self._request.headers.get("Connection")
+ if self._request.supports_http_1_1():
+ disconnect = connection_header == "close"
+ elif ("Content-Length" in self._request.headers
+ or self._request.method in ("HEAD", "GET")):
+ disconnect = connection_header != "Keep-Alive"
+ else:
+ disconnect = True
+ self._request = None
+ self._request_finished = False
+ if disconnect:
+ self.stream.close()
+ return
+ self.stream.read_until("\r\n\r\n", self._on_headers)
+
+ def _on_headers(self, data):
+ eol = data.find("\r\n")
+ start_line = data[:eol]
+ method, uri, version = start_line.split(" ")
+ if not version.startswith("HTTP/"):
+ raise Exception("Malformed HTTP version in HTTP Request-Line")
+ headers = httputil.HTTPHeaders.parse(data[eol:])
+ self._request = HTTPRequest(
+ connection=self, method=method, uri=uri, version=version,
+ headers=headers, remote_ip=self.address[0])
+
+ content_length = headers.get("Content-Length")
+ if content_length:
+ content_length = int(content_length)
+ if content_length > self.stream.max_buffer_size:
+ raise Exception("Content-Length too long")
+ if headers.get("Expect") == "100-continue":
+ self.stream.write("HTTP/1.1 100 (Continue)\r\n\r\n")
+ self.stream.read_bytes(content_length, self._on_request_body)
+ return
+
+ self.request_callback(self._request)
+
+ def _on_request_body(self, data):
+ self._request.body = data
+ content_type = self._request.headers.get("Content-Type", "")
+ if self._request.method == "POST":
+ if content_type.startswith("application/x-www-form-urlencoded"):
+ arguments = cgi.parse_qs(self._request.body)
+ for name, values in arguments.iteritems():
+ values = [v for v in values if v]
+ if values:
+ self._request.arguments.setdefault(name, []).extend(
+ values)
+ elif content_type.startswith("multipart/form-data"):
+ if 'boundary=' in content_type:
+ boundary = content_type.split('boundary=',1)[1]
+ if boundary: self._parse_mime_body(boundary, data)
+ else:
+ logging.warning("Invalid multipart/form-data")
+ self.request_callback(self._request)
+
+ def _parse_mime_body(self, boundary, data):
+ # The standard allows for the boundary to be quoted in the header,
+ # although it's rare (it happens at least for google app engine
+ # xmpp). I think we're also supposed to handle backslash-escapes
+ # here but I'll save that until we see a client that uses them
+ # in the wild.
+ if boundary.startswith('"') and boundary.endswith('"'):
+ boundary = boundary[1:-1]
+ if data.endswith("\r\n"):
+ footer_length = len(boundary) + 6
+ else:
+ footer_length = len(boundary) + 4
+ parts = data[:-footer_length].split("--" + boundary + "\r\n")
+ for part in parts:
+ if not part: continue
+ eoh = part.find("\r\n\r\n")
+ if eoh == -1:
+ logging.warning("multipart/form-data missing headers")
+ continue
+ headers = httputil.HTTPHeaders.parse(part[:eoh])
+ name_header = headers.get("Content-Disposition", "")
+ if not name_header.startswith("form-data;") or \
+ not part.endswith("\r\n"):
+ logging.warning("Invalid multipart/form-data")
+ continue
+ value = part[eoh + 4:-2]
+ name_values = {}
+ for name_part in name_header[10:].split(";"):
+ name, name_value = name_part.strip().split("=", 1)
+ name_values[name] = name_value.strip('"').decode("utf-8")
+ if not name_values.get("name"):
+ logging.warning("multipart/form-data value missing name")
+ continue
+ name = name_values["name"]
+ if name_values.get("filename"):
+ ctype = headers.get("Content-Type", "application/unknown")
+ self._request.files.setdefault(name, []).append(dict(
+ filename=name_values["filename"], body=value,
+ content_type=ctype))
+ else:
+ self._request.arguments.setdefault(name, []).append(value)
+
+
+class HTTPRequest(object):
+ """A single HTTP request.
+
+ GET/POST arguments are available in the arguments property, which
+ maps arguments names to lists of values (to support multiple values
+ for individual names). Names and values are both unicode always.
+
+ File uploads are available in the files property, which maps file
+ names to list of files. Each file is a dictionary of the form
+ {"filename":..., "content_type":..., "body":...}. The content_type
+ comes from the provided HTTP header and should not be trusted
+ outright given that it can be easily forged.
+
+ An HTTP request is attached to a single HTTP connection, which can
+ be accessed through the "connection" attribute. Since connections
+ are typically kept open in HTTP/1.1, multiple requests can be handled
+ sequentially on a single connection.
+ """
+ def __init__(self, method, uri, version="HTTP/1.0", headers=None,
+ body=None, remote_ip=None, protocol=None, host=None,
+ files=None, connection=None):
+ self.method = method
+ self.uri = uri
+ self.version = version
+ self.headers = headers or httputil.HTTPHeaders()
+ self.body = body or ""
+ if connection and connection.xheaders:
+ # Squid uses X-Forwarded-For, others use X-Real-Ip
+ self.remote_ip = self.headers.get(
+ "X-Real-Ip", self.headers.get("X-Forwarded-For", remote_ip))
+ self.protocol = self.headers.get("X-Scheme", protocol) or "http"
+ else:
+ self.remote_ip = remote_ip
+ self.protocol = protocol or "http"
+ self.host = host or self.headers.get("Host") or "127.0.0.1"
+ self.files = files or {}
+ self.connection = connection
+ self._start_time = time.time()
+ self._finish_time = None
+
+ scheme, netloc, path, query, fragment = urlparse.urlsplit(uri)
+ self.path = path
+ self.query = query
+ arguments = cgi.parse_qs(query)
+ self.arguments = {}
+ for name, values in arguments.iteritems():
+ values = [v for v in values if v]
+ if values: self.arguments[name] = values
+
+ def supports_http_1_1(self):
+ """Returns True if this request supports HTTP/1.1 semantics"""
+ return self.version == "HTTP/1.1"
+
+ def write(self, chunk):
+ """Writes the given chunk to the response stream."""
+ assert isinstance(chunk, str)
+ self.connection.write(chunk)
+
+ def finish(self):
+ """Finishes this HTTP request on the open connection."""
+ self.connection.finish()
+ self._finish_time = time.time()
+
+ def full_url(self):
+ """Reconstructs the full URL for this request."""
+ return self.protocol + "://" + self.host + self.uri
+
+ def request_time(self):
+ """Returns the amount of time it took for this request to execute."""
+ if self._finish_time is None:
+ return time.time() - self._start_time
+ else:
+ return self._finish_time - self._start_time
+
+ def __repr__(self):
+ attrs = ("protocol", "host", "method", "uri", "version", "remote_ip",
+ "remote_ip", "body")
+ args = ", ".join(["%s=%r" % (n, getattr(self, n)) for n in attrs])
+ return "%s(%s, headers=%s)" % (
+ self.__class__.__name__, args, dict(self.headers))
+
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""HTTP utility code shared by clients and servers."""
+
+class HTTPHeaders(dict):
+ """A dictionary that maintains Http-Header-Case for all keys.
+
+ Supports multiple values per key via a pair of new methods,
+ add() and get_list(). The regular dictionary interface returns a single
+ value per key, with multiple values joined by a comma.
+
+ >>> h = HTTPHeaders({"content-type": "text/html"})
+ >>> h.keys()
+ ['Content-Type']
+ >>> h["Content-Type"]
+ 'text/html'
+
+ >>> h.add("Set-Cookie", "A=B")
+ >>> h.add("Set-Cookie", "C=D")
+ >>> h["set-cookie"]
+ 'A=B,C=D'
+ >>> h.get_list("set-cookie")
+ ['A=B', 'C=D']
+
+ >>> for (k,v) in sorted(h.get_all()):
+ ... print '%s: %s' % (k,v)
+ ...
+ Content-Type: text/html
+ Set-Cookie: A=B
+ Set-Cookie: C=D
+ """
+ def __init__(self, *args, **kwargs):
+ # Don't pass args or kwargs to dict.__init__, as it will bypass
+ # our __setitem__
+ dict.__init__(self)
+ self._as_list = {}
+ self.update(*args, **kwargs)
+
+ # new public methods
+
+ def add(self, name, value):
+ """Adds a new value for the given key."""
+ norm_name = HTTPHeaders._normalize_name(name)
+ if norm_name in self:
+ # bypass our override of __setitem__ since it modifies _as_list
+ dict.__setitem__(self, norm_name, self[norm_name] + ',' + value)
+ self._as_list[norm_name].append(value)
+ else:
+ self[norm_name] = value
+
+ def get_list(self, name):
+ """Returns all values for the given header as a list."""
+ norm_name = HTTPHeaders._normalize_name(name)
+ return self._as_list.get(norm_name, [])
+
+ def get_all(self):
+ """Returns an iterable of all (name, value) pairs.
+
+ If a header has multiple values, multiple pairs will be
+ returned with the same name.
+ """
+ for name, list in self._as_list.iteritems():
+ for value in list:
+ yield (name, value)
+
+ def parse_line(self, line):
+ """Updates the dictionary with a single header line.
+
+ >>> h = HTTPHeaders()
+ >>> h.parse_line("Content-Type: text/html")
+ >>> h.get('content-type')
+ 'text/html'
+ """
+ name, value = line.split(":", 1)
+ self.add(name, value.strip())
+
+ @classmethod
+ def parse(cls, headers):
+ """Returns a dictionary from HTTP header text.
+
+ >>> h = HTTPHeaders.parse("Content-Type: text/html\\r\\nContent-Length: 42\\r\\n")
+ >>> sorted(h.iteritems())
+ [('Content-Length', '42'), ('Content-Type', 'text/html')]
+ """
+ h = cls()
+ for line in headers.splitlines():
+ if line:
+ h.parse_line(line)
+ return h
+
+ # dict implementation overrides
+
+ def __setitem__(self, name, value):
+ norm_name = HTTPHeaders._normalize_name(name)
+ dict.__setitem__(self, norm_name, value)
+ self._as_list[norm_name] = [value]
+
+ def __getitem__(self, name):
+ return dict.__getitem__(self, HTTPHeaders._normalize_name(name))
+
+ def __delitem__(self, name):
+ norm_name = HTTPHeaders._normalize_name(name)
+ dict.__delitem__(self, norm_name)
+ del self._as_list[norm_name]
+
+ def get(self, name, default=None):
+ return dict.get(self, HTTPHeaders._normalize_name(name), default)
+
+ def update(self, *args, **kwargs):
+ # dict.update bypasses our __setitem__
+ for k, v in dict(*args, **kwargs).iteritems():
+ self[k] = v
+
+ @staticmethod
+ def _normalize_name(name):
+ """Converts a name to Http-Header-Case.
+
+ >>> HTTPHeaders._normalize_name("coNtent-TYPE")
+ 'Content-Type'
+ """
+ return "-".join([w.capitalize() for w in name.split("-")])
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod()
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""A level-triggered I/O loop for non-blocking sockets."""
+
+import bisect
+import errno
+import os
+import logging
+import select
+import time
+import traceback
+
+try:
+ import signal
+except ImportError:
+ signal = None
+
+try:
+ import fcntl
+except ImportError:
+ if os.name == 'nt':
+ import win32_support
+ import win32_support as fcntl
+ else:
+ raise
+
+class IOLoop(object):
+ """A level-triggered I/O loop.
+
+ We use epoll if it is available, or else we fall back on select(). If
+ you are implementing a system that needs to handle 1000s of simultaneous
+ connections, you should use Linux and either compile our epoll module or
+ use Python 2.6+ to get epoll support.
+
+ Example usage for a simple TCP server:
+
+ import errno
+ import functools
+ import ioloop
+ import socket
+
+ def connection_ready(sock, fd, events):
+ while True:
+ try:
+ connection, address = sock.accept()
+ except socket.error, e:
+ if e[0] not in (errno.EWOULDBLOCK, errno.EAGAIN):
+ raise
+ return
+ connection.setblocking(0)
+ handle_connection(connection, address)
+
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ sock.setblocking(0)
+ sock.bind(("", port))
+ sock.listen(128)
+
+ io_loop = ioloop.IOLoop.instance()
+ callback = functools.partial(connection_ready, sock)
+ io_loop.add_handler(sock.fileno(), callback, io_loop.READ)
+ io_loop.start()
+
+ """
+ # Constants from the epoll module
+ _EPOLLIN = 0x001
+ _EPOLLPRI = 0x002
+ _EPOLLOUT = 0x004
+ _EPOLLERR = 0x008
+ _EPOLLHUP = 0x010
+ _EPOLLRDHUP = 0x2000
+ _EPOLLONESHOT = (1 << 30)
+ _EPOLLET = (1 << 31)
+
+ # Our events map exactly to the epoll events
+ NONE = 0
+ READ = _EPOLLIN
+ WRITE = _EPOLLOUT
+ ERROR = _EPOLLERR | _EPOLLHUP | _EPOLLRDHUP
+
+ def __init__(self, impl=None):
+ self._impl = impl or _poll()
+ if hasattr(self._impl, 'fileno'):
+ self._set_close_exec(self._impl.fileno())
+ self._handlers = {}
+ self._events = {}
+ self._callbacks = set()
+ self._timeouts = []
+ self._running = False
+ self._stopped = False
+ self._blocking_log_threshold = None
+
+ # Create a pipe that we send bogus data to when we want to wake
+ # the I/O loop when it is idle
+ if os.name != 'nt':
+ r, w = os.pipe()
+ self._set_nonblocking(r)
+ self._set_nonblocking(w)
+ self._set_close_exec(r)
+ self._set_close_exec(w)
+ self._waker_reader = os.fdopen(r, "r", 0)
+ self._waker_writer = os.fdopen(w, "w", 0)
+ else:
+ self._waker_reader = self._waker_writer = win32_support.Pipe()
+ r = self._waker_writer.reader_fd
+ self.add_handler(r, self._read_waker, self.READ)
+
+ @classmethod
+ def instance(cls):
+ """Returns a global IOLoop instance.
+
+ Most single-threaded applications have a single, global IOLoop.
+ Use this method instead of passing around IOLoop instances
+ throughout your code.
+
+ A common pattern for classes that depend on IOLoops is to use
+ a default argument to enable programs with multiple IOLoops
+ but not require the argument for simpler applications:
+
+ class MyClass(object):
+ def __init__(self, io_loop=None):
+ self.io_loop = io_loop or IOLoop.instance()
+ """
+ if not hasattr(cls, "_instance"):
+ cls._instance = cls()
+ return cls._instance
+
+ @classmethod
+ def initialized(cls):
+ return hasattr(cls, "_instance")
+
+ def add_handler(self, fd, handler, events):
+ """Registers the given handler to receive the given events for fd."""
+ self._handlers[fd] = handler
+ self._impl.register(fd, events | self.ERROR)
+
+ def update_handler(self, fd, events):
+ """Changes the events we listen for fd."""
+ self._impl.modify(fd, events | self.ERROR)
+
+ def remove_handler(self, fd):
+ """Stop listening for events on fd."""
+ self._handlers.pop(fd, None)
+ self._events.pop(fd, None)
+ try:
+ self._impl.unregister(fd)
+ except (OSError, IOError):
+ logging.debug("Error deleting fd from IOLoop", exc_info=True)
+
+ def set_blocking_log_threshold(self, s):
+ """Logs a stack trace if the ioloop is blocked for more than s seconds.
+ Pass None to disable. Requires python 2.6 on a unixy platform.
+ """
+ if not hasattr(signal, "setitimer"):
+ logging.error("set_blocking_log_threshold requires a signal module "
+ "with the setitimer method")
+ return
+ self._blocking_log_threshold = s
+ if s is not None:
+ signal.signal(signal.SIGALRM, self._handle_alarm)
+
+ def _handle_alarm(self, signal, frame):
+ logging.warning('IOLoop blocked for %f seconds in\n%s',
+ self._blocking_log_threshold,
+ ''.join(traceback.format_stack(frame)))
+
+ def start(self):
+ """Starts the I/O loop.
+
+ The loop will run until one of the I/O handlers calls stop(), which
+ will make the loop stop after the current event iteration completes.
+ """
+ if self._stopped:
+ self._stopped = False
+ return
+ self._running = True
+ while True:
+ # Never use an infinite timeout here - it can stall epoll
+ poll_timeout = 0.2
+
+ # Prevent IO event starvation by delaying new callbacks
+ # to the next iteration of the event loop.
+ callbacks = list(self._callbacks)
+ for callback in callbacks:
+ # A callback can add or remove other callbacks
+ if callback in self._callbacks:
+ self._callbacks.remove(callback)
+ self._run_callback(callback)
+
+ if self._callbacks:
+ poll_timeout = 0.0
+
+ if self._timeouts:
+ now = time.time()
+ while self._timeouts and self._timeouts[0].deadline <= now:
+ timeout = self._timeouts.pop(0)
+ self._run_callback(timeout.callback)
+ if self._timeouts:
+ milliseconds = self._timeouts[0].deadline - now
+ poll_timeout = min(milliseconds, poll_timeout)
+
+ if not self._running:
+ break
+
+ if self._blocking_log_threshold is not None:
+ # clear alarm so it doesn't fire while poll is waiting for
+ # events.
+ signal.setitimer(signal.ITIMER_REAL, 0, 0)
+
+ try:
+ event_pairs = self._impl.poll(poll_timeout)
+ except Exception, e:
+ if hasattr(e, 'errno') and e.errno == errno.EINTR:
+ logging.warning("Interrupted system call", exc_info=1)
+ continue
+ else:
+ raise
+
+ if self._blocking_log_threshold is not None:
+ signal.setitimer(signal.ITIMER_REAL,
+ self._blocking_log_threshold, 0)
+
+ # Pop one fd at a time from the set of pending fds and run
+ # its handler. Since that handler may perform actions on
+ # other file descriptors, there may be reentrant calls to
+ # this IOLoop that update self._events
+ self._events.update(event_pairs)
+ while self._events:
+ fd, events = self._events.popitem()
+ try:
+ self._handlers[fd](fd, events)
+ except (KeyboardInterrupt, SystemExit):
+ raise
+ except (OSError, IOError), e:
+ if e[0] == errno.EPIPE:
+ # Happens when the client closes the connection
+ pass
+ else:
+ logging.error("Exception in I/O handler for fd %d",
+ fd, exc_info=True)
+ except:
+ logging.error("Exception in I/O handler for fd %d",
+ fd, exc_info=True)
+ # reset the stopped flag so another start/stop pair can be issued
+ self._stopped = False
+ if self._blocking_log_threshold is not None:
+ signal.setitimer(signal.ITIMER_REAL, 0, 0)
+
+ def stop(self):
+ """Stop the loop after the current event loop iteration is complete.
+ If the event loop is not currently running, the next call to start()
+ will return immediately.
+
+ To use asynchronous methods from otherwise-synchronous code (such as
+ unit tests), you can start and stop the event loop like this:
+ ioloop = IOLoop()
+ async_method(ioloop=ioloop, callback=ioloop.stop)
+ ioloop.start()
+ ioloop.start() will return after async_method has run its callback,
+ whether that callback was invoked before or after ioloop.start.
+ """
+ self._running = False
+ self._stopped = True
+ self._wake()
+
+ def running(self):
+ """Returns true if this IOLoop is currently running."""
+ return self._running
+
+ def add_timeout(self, deadline, callback):
+ """Calls the given callback at the time deadline from the I/O loop."""
+ timeout = _Timeout(deadline, callback)
+ bisect.insort(self._timeouts, timeout)
+ return timeout
+
+ def remove_timeout(self, timeout):
+ self._timeouts.remove(timeout)
+
+ def add_callback(self, callback):
+ """Calls the given callback on the next I/O loop iteration."""
+ self._callbacks.add(callback)
+ self._wake()
+
+ def remove_callback(self, callback):
+ """Removes the given callback from the next I/O loop iteration."""
+ self._callbacks.remove(callback)
+
+ def _wake(self):
+ try:
+ self._waker_writer.write("x")
+ except IOError:
+ pass
+
+ def _run_callback(self, callback):
+ try:
+ callback()
+ except (KeyboardInterrupt, SystemExit):
+ raise
+ except:
+ self.handle_callback_exception(callback)
+
+ def handle_callback_exception(self, callback):
+ """This method is called whenever a callback run by the IOLoop
+ throws an exception.
+
+ By default simply logs the exception as an error. Subclasses
+ may override this method to customize reporting of exceptions.
+
+ The exception itself is not passed explicitly, but is available
+ in sys.exc_info.
+ """
+ logging.error("Exception in callback %r", callback, exc_info=True)
+
+ def _read_waker(self, fd, events):
+ try:
+ while True:
+ self._waker_reader.read()
+ except IOError:
+ pass
+
+ def _set_nonblocking(self, fd):
+ flags = fcntl.fcntl(fd, fcntl.F_GETFL)
+ fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+ def _set_close_exec(self, fd):
+ flags = fcntl.fcntl(fd, fcntl.F_GETFD)
+ fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC)
+
+
+class _Timeout(object):
+ """An IOLoop timeout, a UNIX timestamp and a callback"""
+
+ # Reduce memory overhead when there are lots of pending callbacks
+ __slots__ = ['deadline', 'callback']
+
+ def __init__(self, deadline, callback):
+ self.deadline = deadline
+ self.callback = callback
+
+ def __cmp__(self, other):
+ return cmp((self.deadline, id(self.callback)),
+ (other.deadline, id(other.callback)))
+
+
+class PeriodicCallback(object):
+ """Schedules the given callback to be called periodically.
+
+ The callback is called every callback_time milliseconds.
+ """
+ def __init__(self, callback, callback_time, io_loop=None):
+ self.callback = callback
+ self.callback_time = callback_time
+ self.io_loop = io_loop or IOLoop.instance()
+ self._running = True
+
+ def start(self):
+ timeout = time.time() + self.callback_time / 1000.0
+ self.io_loop.add_timeout(timeout, self._run)
+
+ def stop(self):
+ self._running = False
+
+ def _run(self):
+ if not self._running: return
+ try:
+ self.callback()
+ except (KeyboardInterrupt, SystemExit):
+ raise
+ except:
+ logging.error("Error in periodic callback", exc_info=True)
+ self.start()
+
+
+class _EPoll(object):
+ """An epoll-based event loop using our C module for Python 2.5 systems"""
+ _EPOLL_CTL_ADD = 1
+ _EPOLL_CTL_DEL = 2
+ _EPOLL_CTL_MOD = 3
+
+ def __init__(self):
+ self._epoll_fd = epoll.epoll_create()
+
+ def fileno(self):
+ return self._epoll_fd
+
+ def register(self, fd, events):
+ epoll.epoll_ctl(self._epoll_fd, self._EPOLL_CTL_ADD, fd, events)
+
+ def modify(self, fd, events):
+ epoll.epoll_ctl(self._epoll_fd, self._EPOLL_CTL_MOD, fd, events)
+
+ def unregister(self, fd):
+ epoll.epoll_ctl(self._epoll_fd, self._EPOLL_CTL_DEL, fd, 0)
+
+ def poll(self, timeout):
+ return epoll.epoll_wait(self._epoll_fd, int(timeout * 1000))
+
+
+class _KQueue(object):
+ """A kqueue-based event loop for BSD/Mac systems."""
+ def __init__(self):
+ self._kqueue = select.kqueue()
+ self._active = {}
+
+ def fileno(self):
+ return self._kqueue.fileno()
+
+ def register(self, fd, events):
+ self._control(fd, events, select.KQ_EV_ADD)
+ self._active[fd] = events
+
+ def modify(self, fd, events):
+ self.unregister(fd)
+ self.register(fd, events)
+
+ def unregister(self, fd):
+ events = self._active.pop(fd)
+ self._control(fd, events, select.KQ_EV_DELETE)
+
+ def _control(self, fd, events, flags):
+ kevents = []
+ if events & IOLoop.WRITE:
+ kevents.append(select.kevent(
+ fd, filter=select.KQ_FILTER_WRITE, flags=flags))
+ if events & IOLoop.READ or not kevents:
+ # Always read when there is not a write
+ kevents.append(select.kevent(
+ fd, filter=select.KQ_FILTER_READ, flags=flags))
+ # Even though control() takes a list, it seems to return EINVAL
+ # on Mac OS X (10.6) when there is more than one event in the list.
+ for kevent in kevents:
+ self._kqueue.control([kevent], 0)
+
+ def poll(self, timeout):
+ kevents = self._kqueue.control(None, 1000, timeout)
+ events = {}
+ for kevent in kevents:
+ fd = kevent.ident
+ flags = 0
+ if kevent.filter == select.KQ_FILTER_READ:
+ events[fd] = events.get(fd, 0) | IOLoop.READ
+ if kevent.filter == select.KQ_FILTER_WRITE:
+ events[fd] = events.get(fd, 0) | IOLoop.WRITE
+ if kevent.flags & select.KQ_EV_ERROR:
+ events[fd] = events.get(fd, 0) | IOLoop.ERROR
+ return events.items()
+
+
+class _Select(object):
+ """A simple, select()-based IOLoop implementation for non-Linux systems"""
+ def __init__(self):
+ self.read_fds = set()
+ self.write_fds = set()
+ self.error_fds = set()
+ self.fd_sets = (self.read_fds, self.write_fds, self.error_fds)
+
+ def register(self, fd, events):
+ if events & IOLoop.READ: self.read_fds.add(fd)
+ if events & IOLoop.WRITE: self.write_fds.add(fd)
+ if events & IOLoop.ERROR: self.error_fds.add(fd)
+
+ def modify(self, fd, events):
+ self.unregister(fd)
+ self.register(fd, events)
+
+ def unregister(self, fd):
+ self.read_fds.discard(fd)
+ self.write_fds.discard(fd)
+ self.error_fds.discard(fd)
+
+ def poll(self, timeout):
+ readable, writeable, errors = select.select(
+ self.read_fds, self.write_fds, self.error_fds, timeout)
+ events = {}
+ for fd in readable:
+ events[fd] = events.get(fd, 0) | IOLoop.READ
+ for fd in writeable:
+ events[fd] = events.get(fd, 0) | IOLoop.WRITE
+ for fd in errors:
+ events[fd] = events.get(fd, 0) | IOLoop.ERROR
+ return events.items()
+
+
+# Choose a poll implementation. Use epoll if it is available, fall back to
+# select() for non-Linux platforms
+if hasattr(select, "epoll"):
+ # Python 2.6+ on Linux
+ _poll = select.epoll
+elif hasattr(select, "kqueue"):
+ # Python 2.6+ on BSD or Mac
+ _poll = _KQueue
+else:
+ try:
+ # Linux systems with our C module installed
+ import epoll
+ _poll = _EPoll
+ except:
+ # All other systems
+ import sys
+ if "linux" in sys.platform:
+ logging.warning("epoll module not found; using select()")
+ _poll = _Select
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""A utility class to write to and read from a non-blocking socket."""
+
+import errno
+import ioloop
+import logging
+import socket
+
+class IOStream(object):
+ """A utility class to write to and read from a non-blocking socket.
+
+ We support three methods: write(), read_until(), and read_bytes().
+ All of the methods take callbacks (since writing and reading are
+ non-blocking and asynchronous). read_until() reads the socket until
+ a given delimiter, and read_bytes() reads until a specified number
+ of bytes have been read from the socket.
+
+ A very simple (and broken) HTTP client using this class:
+
+ import ioloop
+ import iostream
+ import socket
+
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
+ s.connect(("friendfeed.com", 80))
+ stream = IOStream(s)
+
+ def on_headers(data):
+ headers = {}
+ for line in data.split("\r\n"):
+ parts = line.split(":")
+ if len(parts) == 2:
+ headers[parts[0].strip()] = parts[1].strip()
+ stream.read_bytes(int(headers["Content-Length"]), on_body)
+
+ def on_body(data):
+ print data
+ stream.close()
+ ioloop.IOLoop.instance().stop()
+
+ stream.write("GET / HTTP/1.0\r\n\r\n")
+ stream.read_until("\r\n\r\n", on_headers)
+ ioloop.IOLoop.instance().start()
+
+ """
+ def __init__(self, socket, io_loop=None, max_buffer_size=104857600,
+ read_chunk_size=4096):
+ self.socket = socket
+ self.socket.setblocking(False)
+ self.io_loop = io_loop or ioloop.IOLoop.instance()
+ self.max_buffer_size = max_buffer_size
+ self.read_chunk_size = read_chunk_size
+ self._read_buffer = ""
+ self._write_buffer = ""
+ self._read_delimiter = None
+ self._read_bytes = None
+ self._read_callback = None
+ self._write_callback = None
+ self._close_callback = None
+ self._state = self.io_loop.ERROR
+ self.io_loop.add_handler(
+ self.socket.fileno(), self._handle_events, self._state)
+
+ def read_until(self, delimiter, callback):
+ """Call callback when we read the given delimiter."""
+ assert not self._read_callback, "Already reading"
+ loc = self._read_buffer.find(delimiter)
+ if loc != -1:
+ self._run_callback(callback, self._consume(loc + len(delimiter)))
+ return
+ self._check_closed()
+ self._read_delimiter = delimiter
+ self._read_callback = callback
+ self._add_io_state(self.io_loop.READ)
+
+ def read_bytes(self, num_bytes, callback):
+ """Call callback when we read the given number of bytes."""
+ assert not self._read_callback, "Already reading"
+ if len(self._read_buffer) >= num_bytes:
+ callback(self._consume(num_bytes))
+ return
+ self._check_closed()
+ self._read_bytes = num_bytes
+ self._read_callback = callback
+ self._add_io_state(self.io_loop.READ)
+
+ def write(self, data, callback=None):
+ """Write the given data to this stream.
+
+ If callback is given, we call it when all of the buffered write
+ data has been successfully written to the stream. If there was
+ previously buffered write data and an old write callback, that
+ callback is simply overwritten with this new callback.
+ """
+ self._check_closed()
+ self._write_buffer += data
+ self._add_io_state(self.io_loop.WRITE)
+ self._write_callback = callback
+
+ def set_close_callback(self, callback):
+ """Call the given callback when the stream is closed."""
+ self._close_callback = callback
+
+ def close(self):
+ """Close this stream."""
+ if self.socket is not None:
+ self.io_loop.remove_handler(self.socket.fileno())
+ self.socket.close()
+ self.socket = None
+ if self._close_callback:
+ self._run_callback(self._close_callback)
+
+ def reading(self):
+ """Returns true if we are currently reading from the stream."""
+ return self._read_callback is not None
+
+ def writing(self):
+ """Returns true if we are currently writing to the stream."""
+ return len(self._write_buffer) > 0
+
+ def closed(self):
+ return self.socket is None
+
+ def _handle_events(self, fd, events):
+ if not self.socket:
+ logging.warning("Got events for closed stream %d", fd)
+ return
+ if events & self.io_loop.READ:
+ self._handle_read()
+ if not self.socket:
+ return
+ if events & self.io_loop.WRITE:
+ self._handle_write()
+ if not self.socket:
+ return
+ if events & self.io_loop.ERROR:
+ self.close()
+ return
+ state = self.io_loop.ERROR
+ if self._read_delimiter or self._read_bytes:
+ state |= self.io_loop.READ
+ if self._write_buffer:
+ state |= self.io_loop.WRITE
+ if state != self._state:
+ self._state = state
+ self.io_loop.update_handler(self.socket.fileno(), self._state)
+
+ def _run_callback(self, callback, *args, **kwargs):
+ try:
+ callback(*args, **kwargs)
+ except:
+ # Close the socket on an uncaught exception from a user callback
+ # (It would eventually get closed when the socket object is
+ # gc'd, but we don't want to rely on gc happening before we
+ # run out of file descriptors)
+ self.close()
+ # Re-raise the exception so that IOLoop.handle_callback_exception
+ # can see it and log the error
+ raise
+
+ def _handle_read(self):
+ try:
+ chunk = self.socket.recv(self.read_chunk_size)
+ except socket.error, e:
+ if e[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
+ return
+ else:
+ logging.warning("Read error on %d: %s",
+ self.socket.fileno(), e)
+ self.close()
+ return
+ if not chunk:
+ self.close()
+ return
+ self._read_buffer += chunk
+ if len(self._read_buffer) >= self.max_buffer_size:
+ logging.error("Reached maximum read buffer size")
+ self.close()
+ return
+ if self._read_bytes:
+ if len(self._read_buffer) >= self._read_bytes:
+ num_bytes = self._read_bytes
+ callback = self._read_callback
+ self._read_callback = None
+ self._read_bytes = None
+ self._run_callback(callback, self._consume(num_bytes))
+ elif self._read_delimiter:
+ loc = self._read_buffer.find(self._read_delimiter)
+ if loc != -1:
+ callback = self._read_callback
+ delimiter_len = len(self._read_delimiter)
+ self._read_callback = None
+ self._read_delimiter = None
+ self._run_callback(callback,
+ self._consume(loc + delimiter_len))
+
+ def _handle_write(self):
+ while self._write_buffer:
+ try:
+ num_bytes = self.socket.send(self._write_buffer)
+ self._write_buffer = self._write_buffer[num_bytes:]
+ except socket.error, e:
+ if e[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
+ break
+ else:
+ logging.warning("Write error on %d: %s",
+ self.socket.fileno(), e)
+ self.close()
+ return
+ if not self._write_buffer and self._write_callback:
+ callback = self._write_callback
+ self._write_callback = None
+ self._run_callback(callback)
+
+ def _consume(self, loc):
+ result = self._read_buffer[:loc]
+ self._read_buffer = self._read_buffer[loc:]
+ return result
+
+ def _check_closed(self):
+ if not self.socket:
+ raise IOError("Stream is closed")
+
+ def _add_io_state(self, state):
+ if not self._state & state:
+ self._state = self._state | state
+ self.io_loop.update_handler(self.socket.fileno(), self._state)
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Translation methods for generating localized strings.
+
+To load a locale and generate a translated string:
+
+ user_locale = locale.get("es_LA")
+ print user_locale.translate("Sign out")
+
+locale.get() returns the closest matching locale, not necessarily the
+specific locale you requested. You can support pluralization with
+additional arguments to translate(), e.g.:
+
+ people = [...]
+ message = user_locale.translate(
+ "%(list)s is online", "%(list)s are online", len(people))
+ print message % {"list": user_locale.list(people)}
+
+The first string is chosen if len(people) == 1, otherwise the second
+string is chosen.
+
+Applications should call one of load_translations (which uses a simple
+CSV format) or load_gettext_translations (which uses the .mo format
+supported by gettext and related tools). If neither method is called,
+the locale.translate method will simply return the original string.
+"""
+
+import csv
+import datetime
+import logging
+import os
+
+_default_locale = "en_US"
+_translations = {}
+_supported_locales = frozenset([_default_locale])
+_use_gettext = False
+
+def get(*locale_codes):
+ """Returns the closest match for the given locale codes.
+
+ We iterate over all given locale codes in order. If we have a tight
+ or a loose match for the code (e.g., "en" for "en_US"), we return
+ the locale. Otherwise we move to the next code in the list.
+
+ By default we return en_US if no translations are found for any of
+ the specified locales. You can change the default locale with
+ set_default_locale() below.
+ """
+ return Locale.get_closest(*locale_codes)
+
+
+def set_default_locale(code):
+ """Sets the default locale, used in get_closest_locale().
+
+ The default locale is assumed to be the language used for all strings
+ in the system. The translations loaded from disk are mappings from
+ the default locale to the destination locale. Consequently, you don't
+ need to create a translation file for the default locale.
+ """
+ global _default_locale
+ global _supported_locales
+ _default_locale = code
+ _supported_locales = frozenset(_translations.keys() + [_default_locale])
+
+
+def load_translations(directory):
+ """Loads translations from CSV files in a directory.
+
+ Translations are strings with optional Python-style named placeholders
+ (e.g., "My name is %(name)s") and their associated translations.
+
+ The directory should have translation files of the form LOCALE.csv,
+ e.g. es_GT.csv. The CSV files should have two or three columns: string,
+ translation, and an optional plural indicator. Plural indicators should
+ be one of "plural" or "singular". A given string can have both singular
+ and plural forms. For example "%(name)s liked this" may have a
+ different verb conjugation depending on whether %(name)s is one
+ name or a list of names. There should be two rows in the CSV file for
+ that string, one with plural indicator "singular", and one "plural".
+ For strings with no verbs that would change on translation, simply
+ use "unknown" or the empty string (or don't include the column at all).
+
+ Example translation es_LA.csv:
+
+ "I love you","Te amo"
+ "%(name)s liked this","A %(name)s les gust\xf3 esto","plural"
+ "%(name)s liked this","A %(name)s le gust\xf3 esto","singular"
+
+ """
+ global _translations
+ global _supported_locales
+ _translations = {}
+ for path in os.listdir(directory):
+ if not path.endswith(".csv"): continue
+ locale, extension = path.split(".")
+ if locale not in LOCALE_NAMES:
+ logging.error("Unrecognized locale %r (path: %s)", locale,
+ os.path.join(directory, path))
+ continue
+ f = open(os.path.join(directory, path), "r")
+ _translations[locale] = {}
+ for i, row in enumerate(csv.reader(f)):
+ if not row or len(row) < 2: continue
+ row = [c.decode("utf-8").strip() for c in row]
+ english, translation = row[:2]
+ if len(row) > 2:
+ plural = row[2] or "unknown"
+ else:
+ plural = "unknown"
+ if plural not in ("plural", "singular", "unknown"):
+ logging.error("Unrecognized plural indicator %r in %s line %d",
+ plural, path, i + 1)
+ continue
+ _translations[locale].setdefault(plural, {})[english] = translation
+ f.close()
+ _supported_locales = frozenset(_translations.keys() + [_default_locale])
+ logging.info("Supported locales: %s", sorted(_supported_locales))
+
+def load_gettext_translations(directory, domain):
+ """Loads translations from gettext's locale tree
+
+ Locale tree is similar to system's /usr/share/locale, like:
+
+ {directory}/{lang}/LC_MESSAGES/{domain}.mo
+
+ Three steps are required to have you app translated:
+
+ 1. Generate POT translation file
+ xgettext --language=Python --keyword=_:1,2 -d cyclone file1.py file2.html etc
+
+ 2. Merge against existing POT file:
+ msgmerge old.po cyclone.po > new.po
+
+ 3. Compile:
+ msgfmt cyclone.po -o {directory}/pt_BR/LC_MESSAGES/cyclone.mo
+ """
+ import gettext
+ global _translations
+ global _supported_locales
+ global _use_gettext
+ _translations = {}
+ for lang in os.listdir(directory):
+ if os.path.isfile(os.path.join(directory, lang)): continue
+ try:
+ os.stat(os.path.join(directory, lang, "LC_MESSAGES", domain+".mo"))
+ _translations[lang] = gettext.translation(domain, directory,
+ languages=[lang])
+ except Exception, e:
+ logging.error("Cannot load translation for '%s': %s", lang, str(e))
+ continue
+ _supported_locales = frozenset(_translations.keys() + [_default_locale])
+ _use_gettext = True
+ logging.info("Supported locales: %s", sorted(_supported_locales))
+
+
+def get_supported_locales(cls):
+ """Returns a list of all the supported locale codes."""
+ return _supported_locales
+
+
+class Locale(object):
+ @classmethod
+ def get_closest(cls, *locale_codes):
+ """Returns the closest match for the given locale code."""
+ for code in locale_codes:
+ if not code: continue
+ code = code.replace("-", "_")
+ parts = code.split("_")
+ if len(parts) > 2:
+ continue
+ elif len(parts) == 2:
+ code = parts[0].lower() + "_" + parts[1].upper()
+ if code in _supported_locales:
+ return cls.get(code)
+ if parts[0].lower() in _supported_locales:
+ return cls.get(parts[0].lower())
+ return cls.get(_default_locale)
+
+ @classmethod
+ def get(cls, code):
+ """Returns the Locale for the given locale code.
+
+ If it is not supported, we raise an exception.
+ """
+ if not hasattr(cls, "_cache"):
+ cls._cache = {}
+ if code not in cls._cache:
+ assert code in _supported_locales
+ translations = _translations.get(code, None)
+ if translations is None:
+ locale = CSVLocale(code, {})
+ elif _use_gettext:
+ locale = GettextLocale(code, translations)
+ else:
+ locale = CSVLocale(code, translations)
+ cls._cache[code] = locale
+ return cls._cache[code]
+
+ def __init__(self, code, translations):
+ self.code = code
+ self.name = LOCALE_NAMES.get(code, {}).get("name", u"Unknown")
+ self.rtl = False
+ for prefix in ["fa", "ar", "he"]:
+ if self.code.startswith(prefix):
+ self.rtl = True
+ break
+ self.translations = translations
+
+ # Initialize strings for date formatting
+ _ = self.translate
+ self._months = [
+ _("January"), _("February"), _("March"), _("April"),
+ _("May"), _("June"), _("July"), _("August"),
+ _("September"), _("October"), _("November"), _("December")]
+ self._weekdays = [
+ _("Monday"), _("Tuesday"), _("Wednesday"), _("Thursday"),
+ _("Friday"), _("Saturday"), _("Sunday")]
+
+ def translate(self, message, plural_message=None, count=None):
+ raise NotImplementedError()
+
+ def format_date(self, date, gmt_offset=0, relative=True, shorter=False,
+ full_format=False):
+ """Formats the given date (which should be GMT).
+
+ By default, we return a relative time (e.g., "2 minutes ago"). You
+ can return an absolute date string with relative=False.
+
+ You can force a full format date ("July 10, 1980") with
+ full_format=True.
+ """
+ if self.code.startswith("ru"):
+ relative = False
+ if type(date) in (int, long, float):
+ date = datetime.datetime.utcfromtimestamp(date)
+ now = datetime.datetime.utcnow()
+ # Round down to now. Due to click skew, things are somethings
+ # slightly in the future.
+ if date > now: date = now
+ local_date = date - datetime.timedelta(minutes=gmt_offset)
+ local_now = now - datetime.timedelta(minutes=gmt_offset)
+ local_yesterday = local_now - datetime.timedelta(hours=24)
+ difference = now - date
+ seconds = difference.seconds
+ days = difference.days
+
+ _ = self.translate
+ format = None
+ if not full_format:
+ if relative and days == 0:
+ if seconds < 50:
+ return _("1 second ago", "%(seconds)d seconds ago",
+ seconds) % { "seconds": seconds }
+
+ if seconds < 50 * 60:
+ minutes = round(seconds / 60.0)
+ return _("1 minute ago", "%(minutes)d minutes ago",
+ minutes) % { "minutes": minutes }
+
+ hours = round(seconds / (60.0 * 60))
+ return _("1 hour ago", "%(hours)d hours ago",
+ hours) % { "hours": hours }
+
+ if days == 0:
+ format = _("%(time)s")
+ elif days == 1 and local_date.day == local_yesterday.day and \
+ relative:
+ format = _("yesterday") if shorter else \
+ _("yesterday at %(time)s")
+ elif days < 5:
+ format = _("%(weekday)s") if shorter else \
+ _("%(weekday)s at %(time)s")
+ elif days < 334: # 11mo, since confusing for same month last year
+ format = _("%(month_name)s %(day)s") if shorter else \
+ _("%(month_name)s %(day)s at %(time)s")
+
+ if format is None:
+ format = _("%(month_name)s %(day)s, %(year)s") if shorter else \
+ _("%(month_name)s %(day)s, %(year)s at %(time)s")
+
+ tfhour_clock = self.code not in ("en", "en_US", "zh_CN")
+ if tfhour_clock:
+ str_time = "%d:%02d" % (local_date.hour, local_date.minute)
+ elif self.code == "zh_CN":
+ str_time = "%s%d:%02d" % (
+ (u'\u4e0a\u5348', u'\u4e0b\u5348')[local_date.hour >= 12],
+ local_date.hour % 12 or 12, local_date.minute)
+ else:
+ str_time = "%d:%02d %s" % (
+ local_date.hour % 12 or 12, local_date.minute,
+ ("am", "pm")[local_date.hour >= 12])
+
+ return format % {
+ "month_name": self._months[local_date.month - 1],
+ "weekday": self._weekdays[local_date.weekday()],
+ "day": str(local_date.day),
+ "year": str(local_date.year),
+ "time": str_time
+ }
+
+ def format_day(self, date, gmt_offset=0, dow=True):
+ """Formats the given date as a day of week.
+
+ Example: "Monday, January 22". You can remove the day of week with
+ dow=False.
+ """
+ local_date = date - datetime.timedelta(minutes=gmt_offset)
+ _ = self.translate
+ if dow:
+ return _("%(weekday)s, %(month_name)s %(day)s") % {
+ "month_name": self._months[local_date.month - 1],
+ "weekday": self._weekdays[local_date.weekday()],
+ "day": str(local_date.day),
+ }
+ else:
+ return _("%(month_name)s %(day)s") % {
+ "month_name": self._months[local_date.month - 1],
+ "day": str(local_date.day),
+ }
+
+ def list(self, parts):
+ """Returns a comma-separated list for the given list of parts.
+
+ The format is, e.g., "A, B and C", "A and B" or just "A" for lists
+ of size 1.
+ """
+ _ = self.translate
+ if len(parts) == 0: return ""
+ if len(parts) == 1: return parts[0]
+ comma = u' \u0648 ' if self.code.startswith("fa") else u", "
+ return _("%(commas)s and %(last)s") % {
+ "commas": comma.join(parts[:-1]),
+ "last": parts[len(parts) - 1],
+ }
+
+ def friendly_number(self, value):
+ """Returns a comma-separated number for the given integer."""
+ if self.code not in ("en", "en_US"):
+ return str(value)
+ value = str(value)
+ parts = []
+ while value:
+ parts.append(value[-3:])
+ value = value[:-3]
+ return ",".join(reversed(parts))
+
+class CSVLocale(Locale):
+ """Locale implementation using tornado's CSV translation format."""
+ def translate(self, message, plural_message=None, count=None):
+ """Returns the translation for the given message for this locale.
+
+ If plural_message is given, you must also provide count. We return
+ plural_message when count != 1, and we return the singular form
+ for the given message when count == 1.
+ """
+ if plural_message is not None:
+ assert count is not None
+ if count != 1:
+ message = plural_message
+ message_dict = self.translations.get("plural", {})
+ else:
+ message_dict = self.translations.get("singular", {})
+ else:
+ message_dict = self.translations.get("unknown", {})
+ return message_dict.get(message, message)
+
+class GettextLocale(Locale):
+ """Locale implementation using the gettext module."""
+ def translate(self, message, plural_message=None, count=None):
+ if plural_message is not None:
+ assert count is not None
+ return self.translations.ungettext(message, plural_message, count)
+ else:
+ return self.translations.ugettext(message)
+
+LOCALE_NAMES = {
+ "af_ZA": {"name_en": u"Afrikaans", "name": u"Afrikaans"},
+ "ar_AR": {"name_en": u"Arabic", "name": u"\u0627\u0644\u0639\u0631\u0628\u064a\u0629"},
+ "bg_BG": {"name_en": u"Bulgarian", "name": u"\u0411\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438"},
+ "bn_IN": {"name_en": u"Bengali", "name": u"\u09ac\u09be\u0982\u09b2\u09be"},
+ "bs_BA": {"name_en": u"Bosnian", "name": u"Bosanski"},
+ "ca_ES": {"name_en": u"Catalan", "name": u"Catal\xe0"},
+ "cs_CZ": {"name_en": u"Czech", "name": u"\u010ce\u0161tina"},
+ "cy_GB": {"name_en": u"Welsh", "name": u"Cymraeg"},
+ "da_DK": {"name_en": u"Danish", "name": u"Dansk"},
+ "de_DE": {"name_en": u"German", "name": u"Deutsch"},
+ "el_GR": {"name_en": u"Greek", "name": u"\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac"},
+ "en_GB": {"name_en": u"English (UK)", "name": u"English (UK)"},
+ "en_US": {"name_en": u"English (US)", "name": u"English (US)"},
+ "es_ES": {"name_en": u"Spanish (Spain)", "name": u"Espa\xf1ol (Espa\xf1a)"},
+ "es_LA": {"name_en": u"Spanish", "name": u"Espa\xf1ol"},
+ "et_EE": {"name_en": u"Estonian", "name": u"Eesti"},
+ "eu_ES": {"name_en": u"Basque", "name": u"Euskara"},
+ "fa_IR": {"name_en": u"Persian", "name": u"\u0641\u0627\u0631\u0633\u06cc"},
+ "fi_FI": {"name_en": u"Finnish", "name": u"Suomi"},
+ "fr_CA": {"name_en": u"French (Canada)", "name": u"Fran\xe7ais (Canada)"},
+ "fr_FR": {"name_en": u"French", "name": u"Fran\xe7ais"},
+ "ga_IE": {"name_en": u"Irish", "name": u"Gaeilge"},
+ "gl_ES": {"name_en": u"Galician", "name": u"Galego"},
+ "he_IL": {"name_en": u"Hebrew", "name": u"\u05e2\u05d1\u05e8\u05d9\u05ea"},
+ "hi_IN": {"name_en": u"Hindi", "name": u"\u0939\u093f\u0928\u094d\u0926\u0940"},
+ "hr_HR": {"name_en": u"Croatian", "name": u"Hrvatski"},
+ "hu_HU": {"name_en": u"Hungarian", "name": u"Magyar"},
+ "id_ID": {"name_en": u"Indonesian", "name": u"Bahasa Indonesia"},
+ "is_IS": {"name_en": u"Icelandic", "name": u"\xcdslenska"},
+ "it_IT": {"name_en": u"Italian", "name": u"Italiano"},
+ "ja_JP": {"name_en": u"Japanese", "name": u"\xe6\xe6\xe8"},
+ "ko_KR": {"name_en": u"Korean", "name": u"\xed\xea\xec"},
+ "lt_LT": {"name_en": u"Lithuanian", "name": u"Lietuvi\u0173"},
+ "lv_LV": {"name_en": u"Latvian", "name": u"Latvie\u0161u"},
+ "mk_MK": {"name_en": u"Macedonian", "name": u"\u041c\u0430\u043a\u0435\u0434\u043e\u043d\u0441\u043a\u0438"},
+ "ml_IN": {"name_en": u"Malayalam", "name": u"\u0d2e\u0d32\u0d2f\u0d3e\u0d33\u0d02"},
+ "ms_MY": {"name_en": u"Malay", "name": u"Bahasa Melayu"},
+ "nb_NO": {"name_en": u"Norwegian (bokmal)", "name": u"Norsk (bokm\xe5l)"},
+ "nl_NL": {"name_en": u"Dutch", "name": u"Nederlands"},
+ "nn_NO": {"name_en": u"Norwegian (nynorsk)", "name": u"Norsk (nynorsk)"},
+ "pa_IN": {"name_en": u"Punjabi", "name": u"\u0a2a\u0a70\u0a1c\u0a3e\u0a2c\u0a40"},
+ "pl_PL": {"name_en": u"Polish", "name": u"Polski"},
+ "pt_BR": {"name_en": u"Portuguese (Brazil)", "name": u"Portugu\xeas (Brasil)"},
+ "pt_PT": {"name_en": u"Portuguese (Portugal)", "name": u"Portugu\xeas (Portugal)"},
+ "ro_RO": {"name_en": u"Romanian", "name": u"Rom\xe2n\u0103"},
+ "ru_RU": {"name_en": u"Russian", "name": u"\u0420\u0443\u0441\u0441\u043a\u0438\u0439"},
+ "sk_SK": {"name_en": u"Slovak", "name": u"Sloven\u010dina"},
+ "sl_SI": {"name_en": u"Slovenian", "name": u"Sloven\u0161\u010dina"},
+ "sq_AL": {"name_en": u"Albanian", "name": u"Shqip"},
+ "sr_RS": {"name_en": u"Serbian", "name": u"\u0421\u0440\u043f\u0441\u043a\u0438"},
+ "sv_SE": {"name_en": u"Swedish", "name": u"Svenska"},
+ "sw_KE": {"name_en": u"Swahili", "name": u"Kiswahili"},
+ "ta_IN": {"name_en": u"Tamil", "name": u"\u0ba4\u0bae\u0bbf\u0bb4\u0bcd"},
+ "te_IN": {"name_en": u"Telugu", "name": u"\u0c24\u0c46\u0c32\u0c41\u0c17\u0c41"},
+ "th_TH": {"name_en": u"Thai", "name": u"\u0e20\u0e32\u0e29\u0e32\u0e44\u0e17\u0e22"},
+ "tl_PH": {"name_en": u"Filipino", "name": u"Filipino"},
+ "tr_TR": {"name_en": u"Turkish", "name": u"T\xfcrk\xe7e"},
+ "uk_UA": {"name_en": u"Ukraini ", "name": u"\u0423\u043a\u0440\u0430\u0457\u043d\u0441\u044c\u043a\u0430"},
+ "vi_VN": {"name_en": u"Vietnamese", "name": u"Ti\u1ebfng Vi\u1ec7t"},
+ "zh_CN": {"name_en": u"Chinese (Simplified)", "name": u"\xe4\xe6(\xe7\xe4)"},
+ "zh_HK": {"name_en": u"Chinese (Hong Kong)", "name": u"\xe4\xe6(\xe9\xe6)"},
+ "zh_TW": {"name_en": u"Chinese (Taiwan)", "name": u"\xe4\xe6(\xe5\xe7)"},
+}
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""A command line parsing module that lets modules define their own options.
+
+Each module defines its own options, e.g.,
+
+ from tornado.options import define, options
+
+ define("mysql_host", default="127.0.0.1:3306", help="Main user DB")
+ define("memcache_hosts", default="127.0.0.1:11011", multiple=True,
+ help="Main user memcache servers")
+
+ def connect():
+ db = database.Connection(options.mysql_host)
+ ...
+
+The main() method of your application does not need to be aware of all of
+the options used throughout your program; they are all automatically loaded
+when the modules are loaded. Your main() method can parse the command line
+or parse a config file with:
+
+ import tornado.options
+ tornado.options.parse_config_file("/etc/server.conf")
+ tornado.options.parse_command_line()
+
+Command line formats are what you would expect ("--myoption=myvalue").
+Config files are just Python files. Global names become options, e.g.,
+
+ myoption = "myvalue"
+ myotheroption = "myothervalue"
+
+We support datetimes, timedeltas, ints, and floats (just pass a 'type'
+kwarg to define). We also accept multi-value options. See the documentation
+for define() below.
+"""
+
+import datetime
+import logging
+import logging.handlers
+import re
+import sys
+import time
+
+# For pretty log messages, if available
+try:
+ import curses
+except:
+ curses = None
+
+
+def define(name, default=None, type=str, help=None, metavar=None,
+ multiple=False):
+ """Defines a new command line option.
+
+ If type is given (one of str, float, int, datetime, or timedelta),
+ we parse the command line arguments based on the given type. If
+ multiple is True, we accept comma-separated values, and the option
+ value is always a list.
+
+ For multi-value integers, we also accept the syntax x:y, which
+ turns into range(x, y) - very useful for long integer ranges.
+
+ help and metavar are used to construct the automatically generated
+ command line help string. The help message is formatted like:
+
+ --name=METAVAR help string
+
+ Command line option names must be unique globally. They can be parsed
+ from the command line with parse_command_line() or parsed from a
+ config file with parse_config_file.
+ """
+ if name in options:
+ raise Error("Option %r already defined in %s", name,
+ options[name].file_name)
+ frame = sys._getframe(0)
+ options_file = frame.f_code.co_filename
+ file_name = frame.f_back.f_code.co_filename
+ if file_name == options_file: file_name = ""
+ options[name] = _Option(name, file_name=file_name, default=default,
+ type=type, help=help, metavar=metavar,
+ multiple=multiple)
+
+
+def parse_command_line(args=None):
+ """Parses all options given on the command line.
+
+ We return all command line arguments that are not options as a list.
+ """
+ if args is None: args = sys.argv
+ remaining = []
+ for i in xrange(1, len(args)):
+ # All things after the last option are command line arguments
+ if not args[i].startswith("-"):
+ remaining = args[i:]
+ break
+ if args[i] == "--":
+ remaining = args[i+1:]
+ break
+ arg = args[i].lstrip("-")
+ name, equals, value = arg.partition("=")
+ name = name.replace('-', '_')
+ if not name in options:
+ print_help()
+ raise Error('Unrecognized command line option: %r' % name)
+ option = options[name]
+ if not equals:
+ if option.type == bool:
+ value = "true"
+ else:
+ raise Error('Option %r requires a value' % name)
+ option.parse(value)
+ if options.help:
+ print_help()
+ sys.exit(0)
+
+ # Set up log level and pretty console logging by default
+ if options.logging != 'none':
+ logging.getLogger().setLevel(getattr(logging, options.logging.upper()))
+ enable_pretty_logging()
+
+ return remaining
+
+
+def parse_config_file(path):
+ """Parses and loads the Python config file at the given path."""
+ config = {}
+ execfile(path, config, config)
+ for name in config:
+ if name in options:
+ options[name].set(config[name])
+
+
+def print_help(file=sys.stdout):
+ """Prints all the command line options to stdout."""
+ print >> file, "Usage: %s [OPTIONS]" % sys.argv[0]
+ print >> file, ""
+ print >> file, "Options:"
+ by_file = {}
+ for option in options.itervalues():
+ by_file.setdefault(option.file_name, []).append(option)
+
+ for filename, o in sorted(by_file.items()):
+ if filename: print >> file, filename
+ o.sort(key=lambda option: option.name)
+ for option in o:
+ prefix = option.name
+ if option.metavar:
+ prefix += "=" + option.metavar
+ print >> file, " --%-30s %s" % (prefix, option.help or "")
+ print >> file
+
+
+class _Options(dict):
+ """Our global program options, an dictionary with object-like access."""
+ @classmethod
+ def instance(cls):
+ if not hasattr(cls, "_instance"):
+ cls._instance = cls()
+ return cls._instance
+
+ def __getattr__(self, name):
+ if isinstance(self.get(name), _Option):
+ return self[name].value()
+ raise AttributeError("Unrecognized option %r" % name)
+
+
+class _Option(object):
+ def __init__(self, name, default=None, type=str, help=None, metavar=None,
+ multiple=False, file_name=None):
+ if default is None and multiple:
+ default = []
+ self.name = name
+ self.type = type
+ self.help = help
+ self.metavar = metavar
+ self.multiple = multiple
+ self.file_name = file_name
+ self.default = default
+ self._value = None
+
+ def value(self):
+ return self.default if self._value is None else self._value
+
+ def parse(self, value):
+ _parse = {
+ datetime.datetime: self._parse_datetime,
+ datetime.timedelta: self._parse_timedelta,
+ bool: self._parse_bool,
+ str: self._parse_string,
+ }.get(self.type, self.type)
+ if self.multiple:
+ if self._value is None:
+ self._value = []
+ for part in value.split(","):
+ if self.type in (int, long):
+ # allow ranges of the form X:Y (inclusive at both ends)
+ lo, _, hi = part.partition(":")
+ lo = _parse(lo)
+ hi = _parse(hi) if hi else lo
+ self._value.extend(range(lo, hi+1))
+ else:
+ self._value.append(_parse(part))
+ else:
+ self._value = _parse(value)
+ return self.value()
+
+ def set(self, value):
+ if self.multiple:
+ if not isinstance(value, list):
+ raise Error("Option %r is required to be a list of %s" %
+ (self.name, self.type.__name__))
+ for item in value:
+ if item != None and not isinstance(item, self.type):
+ raise Error("Option %r is required to be a list of %s" %
+ (self.name, self.type.__name__))
+ else:
+ if value != None and not isinstance(value, self.type):
+ raise Error("Option %r is required to be a %s" %
+ (self.name, self.type.__name__))
+ self._value = value
+
+ # Supported date/time formats in our options
+ _DATETIME_FORMATS = [
+ "%a %b %d %H:%M:%S %Y",
+ "%Y-%m-%d %H:%M:%S",
+ "%Y-%m-%d %H:%M",
+ "%Y-%m-%dT%H:%M",
+ "%Y%m%d %H:%M:%S",
+ "%Y%m%d %H:%M",
+ "%Y-%m-%d",
+ "%Y%m%d",
+ "%H:%M:%S",
+ "%H:%M",
+ ]
+
+ def _parse_datetime(self, value):
+ for format in self._DATETIME_FORMATS:
+ try:
+ return datetime.datetime.strptime(value, format)
+ except ValueError:
+ pass
+ raise Error('Unrecognized date/time format: %r' % value)
+
+ _TIMEDELTA_ABBREVS = [
+ ('hours', ['h']),
+ ('minutes', ['m', 'min']),
+ ('seconds', ['s', 'sec']),
+ ('milliseconds', ['ms']),
+ ('microseconds', ['us']),
+ ('days', ['d']),
+ ('weeks', ['w']),
+ ]
+
+ _TIMEDELTA_ABBREV_DICT = dict(
+ (abbrev, full) for full, abbrevs in _TIMEDELTA_ABBREVS
+ for abbrev in abbrevs)
+
+ _FLOAT_PATTERN = r'[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?'
+
+ _TIMEDELTA_PATTERN = re.compile(
+ r'\s*(%s)\s*(\w*)\s*' % _FLOAT_PATTERN, re.IGNORECASE)
+
+ def _parse_timedelta(self, value):
+ try:
+ sum = datetime.timedelta()
+ start = 0
+ while start < len(value):
+ m = self._TIMEDELTA_PATTERN.match(value, start)
+ if not m:
+ raise Exception()
+ num = float(m.group(1))
+ units = m.group(2) or 'seconds'
+ units = self._TIMEDELTA_ABBREV_DICT.get(units, units)
+ sum += datetime.timedelta(**{units: num})
+ start = m.end()
+ return sum
+ except:
+ raise
+
+ def _parse_bool(self, value):
+ return value.lower() not in ("false", "0", "f")
+
+ def _parse_string(self, value):
+ return value.decode("utf-8")
+
+
+class Error(Exception):
+ pass
+
+
+def enable_pretty_logging():
+ """Turns on formatted logging output as configured."""
+ if (options.log_to_stderr or
+ (options.log_to_stderr is None and not options.log_file_prefix)):
+ # Set up color if we are in a tty and curses is installed
+ color = False
+ if curses and sys.stderr.isatty():
+ try:
+ curses.setupterm()
+ if curses.tigetnum("colors") > 0:
+ color = True
+ except:
+ pass
+ channel = logging.StreamHandler()
+ channel.setFormatter(_LogFormatter(color=color))
+ logging.getLogger().addHandler(channel)
+
+ if options.log_file_prefix:
+ channel = logging.handlers.RotatingFileHandler(
+ filename=options.log_file_prefix,
+ maxBytes=options.log_file_max_size,
+ backupCount=options.log_file_num_backups)
+ channel.setFormatter(_LogFormatter(color=False))
+ logging.getLogger().addHandler(channel)
+
+
+class _LogFormatter(logging.Formatter):
+ def __init__(self, color, *args, **kwargs):
+ logging.Formatter.__init__(self, *args, **kwargs)
+ self._color = color
+ if color:
+ fg_color = curses.tigetstr("setaf") or curses.tigetstr("setf") or ""
+ self._colors = {
+ logging.DEBUG: curses.tparm(fg_color, 4), # Blue
+ logging.INFO: curses.tparm(fg_color, 2), # Green
+ logging.WARNING: curses.tparm(fg_color, 3), # Yellow
+ logging.ERROR: curses.tparm(fg_color, 1), # Red
+ }
+ self._normal = curses.tigetstr("sgr0")
+
+ def format(self, record):
+ try:
+ record.message = record.getMessage()
+ except Exception, e:
+ record.message = "Bad message (%r): %r" % (e, record.__dict__)
+ record.asctime = time.strftime(
+ "%y%m%d %H:%M:%S", self.converter(record.created))
+ prefix = '[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]' % \
+ record.__dict__
+ if self._color:
+ prefix = (self._colors.get(record.levelno, self._normal) +
+ prefix + self._normal)
+ formatted = prefix + " " + record.message
+ if record.exc_info:
+ if not record.exc_text:
+ record.exc_text = self.formatException(record.exc_info)
+ if record.exc_text:
+ formatted = formatted.rstrip() + "\n" + record.exc_text
+ return formatted.replace("\n", "\n ")
+
+
+options = _Options.instance()
+
+
+# Default options
+define("help", type=bool, help="show this help information")
+define("logging", default="info",
+ help=("Set the Python log level. If 'none', tornado won't touch the "
+ "logging configuration."),
+ metavar="info|warning|error|none")
+define("log_to_stderr", type=bool, default=None,
+ help=("Send log output to stderr (colorized if possible). "
+ "By default use stderr if --log_file_prefix is not set."))
+define("log_file_prefix", type=str, default=None, metavar="PATH",
+ help=("Path prefix for log files. "
+ "Note that if you are running multiple tornado processes, "
+ "log_file_prefix must be different for each of them (e.g. "
+ "include the port number)"))
+define("log_file_max_size", type=int, default=100 * 1000 * 1000,
+ help="max size of log files before rollover")
+define("log_file_num_backups", type=int, default=10,
+ help="number of log files to keep")
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Implementation of an S3-like storage server based on local files.
+
+Useful to test features that will eventually run on S3, or if you want to
+run something locally that was once running on S3.
+
+We don't support all the features of S3, but it does work with the
+standard S3 client for the most basic semantics. To use the standard
+S3 client with this module:
+
+ c = S3.AWSAuthConnection("", "", server="localhost", port=8888,
+ is_secure=False)
+ c.create_bucket("mybucket")
+ c.put("mybucket", "mykey", "a value")
+ print c.get("mybucket", "mykey").body
+
+"""
+
+import bisect
+import datetime
+import escape
+import hashlib
+import httpserver
+import ioloop
+import os
+import os.path
+import urllib
+import web
+
+
+def start(port, root_directory="/tmp/s3", bucket_depth=0):
+ """Starts the mock S3 server on the given port at the given path."""
+ application = S3Application(root_directory, bucket_depth)
+ http_server = httpserver.HTTPServer(application)
+ http_server.listen(port)
+ ioloop.IOLoop.instance().start()
+
+
+class S3Application(web.Application):
+ """Implementation of an S3-like storage server based on local files.
+
+ If bucket depth is given, we break files up into multiple directories
+ to prevent hitting file system limits for number of files in each
+ directories. 1 means one level of directories, 2 means 2, etc.
+ """
+ def __init__(self, root_directory, bucket_depth=0):
+ web.Application.__init__(self, [
+ (r"/", RootHandler),
+ (r"/([^/]+)/(.+)", ObjectHandler),
+ (r"/([^/]+)/", BucketHandler),
+ ])
+ self.directory = os.path.abspath(root_directory)
+ if not os.path.exists(self.directory):
+ os.makedirs(self.directory)
+ self.bucket_depth = bucket_depth
+
+
+class BaseRequestHandler(web.RequestHandler):
+ SUPPORTED_METHODS = ("PUT", "GET", "DELETE")
+
+ def render_xml(self, value):
+ assert isinstance(value, dict) and len(value) == 1
+ self.set_header("Content-Type", "application/xml; charset=UTF-8")
+ name = value.keys()[0]
+ parts = []
+ parts.append('<' + escape.utf8(name) +
+ ' xmlns="http://doc.s3.amazonaws.com/2006-03-01">')
+ self._render_parts(value.values()[0], parts)
+ parts.append('</' + escape.utf8(name) + '>')
+ self.finish('<?xml version="1.0" encoding="UTF-8"?>\n' +
+ ''.join(parts))
+
+ def _render_parts(self, value, parts=[]):
+ if isinstance(value, basestring):
+ parts.append(escape.xhtml_escape(value))
+ elif isinstance(value, int) or isinstance(value, long):
+ parts.append(str(value))
+ elif isinstance(value, datetime.datetime):
+ parts.append(value.strftime("%Y-%m-%dT%H:%M:%S.000Z"))
+ elif isinstance(value, dict):
+ for name, subvalue in value.iteritems():
+ if not isinstance(subvalue, list):
+ subvalue = [subvalue]
+ for subsubvalue in subvalue:
+ parts.append('<' + escape.utf8(name) + '>')
+ self._render_parts(subsubvalue, parts)
+ parts.append('</' + escape.utf8(name) + '>')
+ else:
+ raise Exception("Unknown S3 value type %r", value)
+
+ def _object_path(self, bucket, object_name):
+ if self.application.bucket_depth < 1:
+ return os.path.abspath(os.path.join(
+ self.application.directory, bucket, object_name))
+ hash = hashlib.md5(object_name).hexdigest()
+ path = os.path.abspath(os.path.join(
+ self.application.directory, bucket))
+ for i in range(self.application.bucket_depth):
+ path = os.path.join(path, hash[:2 * (i + 1)])
+ return os.path.join(path, object_name)
+
+
+class RootHandler(BaseRequestHandler):
+ def get(self):
+ names = os.listdir(self.application.directory)
+ buckets = []
+ for name in names:
+ path = os.path.join(self.application.directory, name)
+ info = os.stat(path)
+ buckets.append({
+ "Name": name,
+ "CreationDate": datetime.datetime.utcfromtimestamp(
+ info.st_ctime),
+ })
+ self.render_xml({"ListAllMyBucketsResult": {
+ "Buckets": {"Bucket": buckets},
+ }})
+
+
+class BucketHandler(BaseRequestHandler):
+ def get(self, bucket_name):
+ prefix = self.get_argument("prefix", u"")
+ marker = self.get_argument("marker", u"")
+ max_keys = int(self.get_argument("max-keys", 50000))
+ path = os.path.abspath(os.path.join(self.application.directory,
+ bucket_name))
+ terse = int(self.get_argument("terse", 0))
+ if not path.startswith(self.application.directory) or \
+ not os.path.isdir(path):
+ raise web.HTTPError(404)
+ object_names = []
+ for root, dirs, files in os.walk(path):
+ for file_name in files:
+ object_names.append(os.path.join(root, file_name))
+ skip = len(path) + 1
+ for i in range(self.application.bucket_depth):
+ skip += 2 * (i + 1) + 1
+ object_names = [n[skip:] for n in object_names]
+ object_names.sort()
+ contents = []
+
+ start_pos = 0
+ if marker:
+ start_pos = bisect.bisect_right(object_names, marker, start_pos)
+ if prefix:
+ start_pos = bisect.bisect_left(object_names, prefix, start_pos)
+
+ truncated = False
+ for object_name in object_names[start_pos:]:
+ if not object_name.startswith(prefix):
+ break
+ if len(contents) >= max_keys:
+ truncated = True
+ break
+ object_path = self._object_path(bucket_name, object_name)
+ c = {"Key": object_name}
+ if not terse:
+ info = os.stat(object_path)
+ c.update({
+ "LastModified": datetime.datetime.utcfromtimestamp(
+ info.st_mtime),
+ "Size": info.st_size,
+ })
+ contents.append(c)
+ marker = object_name
+ self.render_xml({"ListBucketResult": {
+ "Name": bucket_name,
+ "Prefix": prefix,
+ "Marker": marker,
+ "MaxKeys": max_keys,
+ "IsTruncated": truncated,
+ "Contents": contents,
+ }})
+
+ def put(self, bucket_name):
+ path = os.path.abspath(os.path.join(
+ self.application.directory, bucket_name))
+ if not path.startswith(self.application.directory) or \
+ os.path.exists(path):
+ raise web.HTTPError(403)
+ os.makedirs(path)
+ self.finish()
+
+ def delete(self, bucket_name):
+ path = os.path.abspath(os.path.join(
+ self.application.directory, bucket_name))
+ if not path.startswith(self.application.directory) or \
+ not os.path.isdir(path):
+ raise web.HTTPError(404)
+ if len(os.listdir(path)) > 0:
+ raise web.HTTPError(403)
+ os.rmdir(path)
+ self.set_status(204)
+ self.finish()
+
+
+class ObjectHandler(BaseRequestHandler):
+ def get(self, bucket, object_name):
+ object_name = urllib.unquote(object_name)
+ path = self._object_path(bucket, object_name)
+ if not path.startswith(self.application.directory) or \
+ not os.path.isfile(path):
+ raise web.HTTPError(404)
+ info = os.stat(path)
+ self.set_header("Content-Type", "application/unknown")
+ self.set_header("Last-Modified", datetime.datetime.utcfromtimestamp(
+ info.st_mtime))
+ object_file = open(path, "r")
+ try:
+ self.finish(object_file.read())
+ finally:
+ object_file.close()
+
+ def put(self, bucket, object_name):
+ object_name = urllib.unquote(object_name)
+ bucket_dir = os.path.abspath(os.path.join(
+ self.application.directory, bucket))
+ if not bucket_dir.startswith(self.application.directory) or \
+ not os.path.isdir(bucket_dir):
+ raise web.HTTPError(404)
+ path = self._object_path(bucket, object_name)
+ if not path.startswith(bucket_dir) or os.path.isdir(path):
+ raise web.HTTPError(403)
+ directory = os.path.dirname(path)
+ if not os.path.exists(directory):
+ os.makedirs(directory)
+ object_file = open(path, "w")
+ object_file.write(self.request.body)
+ object_file.close()
+ self.finish()
+
+ def delete(self, bucket, object_name):
+ object_name = urllib.unquote(object_name)
+ path = self._object_path(bucket, object_name)
+ if not path.startswith(self.application.directory) or \
+ not os.path.isfile(path):
+ raise web.HTTPError(404)
+ os.unlink(path)
+ self.set_status(204)
+ self.finish()
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""A simple template system that compiles templates to Python code.
+
+Basic usage looks like:
+
+ t = template.Template("<html>{{ myvalue }}</html>")
+ print t.generate(myvalue="XXX")
+
+Loader is a class that loads templates from a root directory and caches
+the compiled templates:
+
+ loader = template.Loader("/home/btaylor")
+ print loader.load("test.html").generate(myvalue="XXX")
+
+We compile all templates to raw Python. Error-reporting is currently... uh,
+interesting. Syntax for the templates
+
+ ### base.html
+ <html>
+ <head>
+ <title>{% block title %}Default title{% end %}</title>
+ </head>
+ <body>
+ <ul>
+ {% for student in students %}
+ {% block student %}
+ <li>{{ escape(student.name) }}</li>
+ {% end %}
+ {% end %}
+ </ul>
+ </body>
+ </html>
+
+ ### bold.html
+ {% extends "base.html" %}
+
+ {% block title %}A bolder title{% end %}
+
+ {% block student %}
+ <li><span style="bold">{{ escape(student.name) }}</span></li>
+ {% block %}
+
+Unlike most other template systems, we do not put any restrictions on the
+expressions you can include in your statements. if and for blocks get
+translated exactly into Python, do you can do complex expressions like:
+
+ {% for student in [p for p in people if p.student and p.age > 23] %}
+ <li>{{ escape(student.name) }}</li>
+ {% end %}
+
+Translating directly to Python means you can apply functions to expressions
+easily, like the escape() function in the examples above. You can pass
+functions in to your template just like any other variable:
+
+ ### Python code
+ def add(x, y):
+ return x + y
+ template.execute(add=add)
+
+ ### The template
+ {{ add(1, 2) }}
+
+We provide the functions escape(), url_escape(), json_encode(), and squeeze()
+to all templates by default.
+"""
+
+from __future__ import with_statement
+
+import cStringIO
+import datetime
+import escape
+import logging
+import os.path
+import re
+
+class Template(object):
+ """A compiled template.
+
+ We compile into Python from the given template_string. You can generate
+ the template from variables with generate().
+ """
+ def __init__(self, template_string, name="<string>", loader=None,
+ compress_whitespace=None):
+ self.name = name
+ if compress_whitespace is None:
+ compress_whitespace = name.endswith(".html") or \
+ name.endswith(".js")
+ reader = _TemplateReader(name, template_string)
+ self.file = _File(_parse(reader))
+ self.code = self._generate_python(loader, compress_whitespace)
+ try:
+ self.compiled = compile(self.code, self.name, "exec")
+ except:
+ formatted_code = _format_code(self.code).rstrip()
+ logging.error("%s code:\n%s", self.name, formatted_code)
+ raise
+
+ def generate(self, **kwargs):
+ """Generate this template with the given arguments."""
+ namespace = {
+ "escape": escape.xhtml_escape,
+ "url_escape": escape.url_escape,
+ "json_encode": escape.json_encode,
+ "squeeze": escape.squeeze,
+ "datetime": datetime,
+ }
+ namespace.update(kwargs)
+ exec self.compiled in namespace
+ execute = namespace["_execute"]
+ try:
+ return execute()
+ except:
+ formatted_code = _format_code(self.code).rstrip()
+ logging.error("%s code:\n%s", self.name, formatted_code)
+ raise
+
+ def _generate_python(self, loader, compress_whitespace):
+ buffer = cStringIO.StringIO()
+ try:
+ named_blocks = {}
+ ancestors = self._get_ancestors(loader)
+ ancestors.reverse()
+ for ancestor in ancestors:
+ ancestor.find_named_blocks(loader, named_blocks)
+ self.file.find_named_blocks(loader, named_blocks)
+ writer = _CodeWriter(buffer, named_blocks, loader, self,
+ compress_whitespace)
+ ancestors[0].generate(writer)
+ return buffer.getvalue()
+ finally:
+ buffer.close()
+
+ def _get_ancestors(self, loader):
+ ancestors = [self.file]
+ for chunk in self.file.body.chunks:
+ if isinstance(chunk, _ExtendsBlock):
+ if not loader:
+ raise ParseError("{% extends %} block found, but no "
+ "template loader")
+ template = loader.load(chunk.name, self.name)
+ ancestors.extend(template._get_ancestors(loader))
+ return ancestors
+
+
+class Loader(object):
+ """A template loader that loads from a single root directory.
+
+ You must use a template loader to use template constructs like
+ {% extends %} and {% include %}. Loader caches all templates after
+ they are loaded the first time.
+ """
+ def __init__(self, root_directory):
+ self.root = os.path.abspath(root_directory)
+ self.templates = {}
+
+ def reset(self):
+ self.templates = {}
+
+ def resolve_path(self, name, parent_path=None):
+ if parent_path and not parent_path.startswith("<") and \
+ not parent_path.startswith("/") and \
+ not name.startswith("/"):
+ current_path = os.path.join(self.root, parent_path)
+ file_dir = os.path.dirname(os.path.abspath(current_path))
+ relative_path = os.path.abspath(os.path.join(file_dir, name))
+ if relative_path.startswith(self.root):
+ name = relative_path[len(self.root) + 1:]
+ return name
+
+ def load(self, name, parent_path=None):
+ name = self.resolve_path(name, parent_path=parent_path)
+ if name not in self.templates:
+ path = os.path.join(self.root, name)
+ f = open(path, "r")
+ self.templates[name] = Template(f.read(), name=name, loader=self)
+ f.close()
+ return self.templates[name]
+
+
+class _Node(object):
+ def each_child(self):
+ return ()
+
+ def generate(self, writer):
+ raise NotImplementedError()
+
+ def find_named_blocks(self, loader, named_blocks):
+ for child in self.each_child():
+ child.find_named_blocks(loader, named_blocks)
+
+
+class _File(_Node):
+ def __init__(self, body):
+ self.body = body
+
+ def generate(self, writer):
+ writer.write_line("def _execute():")
+ with writer.indent():
+ writer.write_line("_buffer = []")
+ self.body.generate(writer)
+ writer.write_line("return ''.join(_buffer)")
+
+ def each_child(self):
+ return (self.body,)
+
+
+
+class _ChunkList(_Node):
+ def __init__(self, chunks):
+ self.chunks = chunks
+
+ def generate(self, writer):
+ for chunk in self.chunks:
+ chunk.generate(writer)
+
+ def each_child(self):
+ return self.chunks
+
+
+class _NamedBlock(_Node):
+ def __init__(self, name, body=None):
+ self.name = name
+ self.body = body
+
+ def each_child(self):
+ return (self.body,)
+
+ def generate(self, writer):
+ writer.named_blocks[self.name].generate(writer)
+
+ def find_named_blocks(self, loader, named_blocks):
+ named_blocks[self.name] = self.body
+ _Node.find_named_blocks(self, loader, named_blocks)
+
+
+class _ExtendsBlock(_Node):
+ def __init__(self, name):
+ self.name = name
+
+
+class _IncludeBlock(_Node):
+ def __init__(self, name, reader):
+ self.name = name
+ self.template_name = reader.name
+
+ def find_named_blocks(self, loader, named_blocks):
+ included = loader.load(self.name, self.template_name)
+ included.file.find_named_blocks(loader, named_blocks)
+
+ def generate(self, writer):
+ included = writer.loader.load(self.name, self.template_name)
+ old = writer.current_template
+ writer.current_template = included
+ included.file.body.generate(writer)
+ writer.current_template = old
+
+
+class _ApplyBlock(_Node):
+ def __init__(self, method, body=None):
+ self.method = method
+ self.body = body
+
+ def each_child(self):
+ return (self.body,)
+
+ def generate(self, writer):
+ method_name = "apply%d" % writer.apply_counter
+ writer.apply_counter += 1
+ writer.write_line("def %s():" % method_name)
+ with writer.indent():
+ writer.write_line("_buffer = []")
+ self.body.generate(writer)
+ writer.write_line("return ''.join(_buffer)")
+ writer.write_line("_buffer.append(%s(%s()))" % (
+ self.method, method_name))
+
+
+class _ControlBlock(_Node):
+ def __init__(self, statement, body=None):
+ self.statement = statement
+ self.body = body
+
+ def each_child(self):
+ return (self.body,)
+
+ def generate(self, writer):
+ writer.write_line("%s:" % self.statement)
+ with writer.indent():
+ self.body.generate(writer)
+
+
+class _IntermediateControlBlock(_Node):
+ def __init__(self, statement):
+ self.statement = statement
+
+ def generate(self, writer):
+ writer.write_line("%s:" % self.statement, writer.indent_size() - 1)
+
+
+class _Statement(_Node):
+ def __init__(self, statement):
+ self.statement = statement
+
+ def generate(self, writer):
+ writer.write_line(self.statement)
+
+
+class _Expression(_Node):
+ def __init__(self, expression):
+ self.expression = expression
+
+ def generate(self, writer):
+ writer.write_line("_tmp = %s" % self.expression)
+ writer.write_line("if isinstance(_tmp, str): _buffer.append(_tmp)")
+ writer.write_line("elif isinstance(_tmp, unicode): "
+ "_buffer.append(_tmp.encode('utf-8'))")
+ writer.write_line("else: _buffer.append(str(_tmp))")
+
+
+class _Text(_Node):
+ def __init__(self, value):
+ self.value = value
+
+ def generate(self, writer):
+ value = self.value
+
+ # Compress lots of white space to a single character. If the whitespace
+ # breaks a line, have it continue to break a line, but just with a
+ # single \n character
+ if writer.compress_whitespace and "<pre>" not in value:
+ value = re.sub(r"([\t ]+)", " ", value)
+ value = re.sub(r"(\s*\n\s*)", "\n", value)
+
+ if value:
+ writer.write_line('_buffer.append(%r)' % value)
+
+
+class ParseError(Exception):
+ """Raised for template syntax errors."""
+ pass
+
+
+class _CodeWriter(object):
+ def __init__(self, file, named_blocks, loader, current_template,
+ compress_whitespace):
+ self.file = file
+ self.named_blocks = named_blocks
+ self.loader = loader
+ self.current_template = current_template
+ self.compress_whitespace = compress_whitespace
+ self.apply_counter = 0
+ self._indent = 0
+
+ def indent(self):
+ return self
+
+ def indent_size(self):
+ return self._indent
+
+ def __enter__(self):
+ self._indent += 1
+ return self
+
+ def __exit__(self, *args):
+ assert self._indent > 0
+ self._indent -= 1
+
+ def write_line(self, line, indent=None):
+ if indent == None:
+ indent = self._indent
+ for i in xrange(indent):
+ self.file.write(" ")
+ print >> self.file, line
+
+
+class _TemplateReader(object):
+ def __init__(self, name, text):
+ self.name = name
+ self.text = text
+ self.line = 0
+ self.pos = 0
+
+ def find(self, needle, start=0, end=None):
+ assert start >= 0, start
+ pos = self.pos
+ start += pos
+ if end is None:
+ index = self.text.find(needle, start)
+ else:
+ end += pos
+ assert end >= start
+ index = self.text.find(needle, start, end)
+ if index != -1:
+ index -= pos
+ return index
+
+ def consume(self, count=None):
+ if count is None:
+ count = len(self.text) - self.pos
+ newpos = self.pos + count
+ self.line += self.text.count("\n", self.pos, newpos)
+ s = self.text[self.pos:newpos]
+ self.pos = newpos
+ return s
+
+ def remaining(self):
+ return len(self.text) - self.pos
+
+ def __len__(self):
+ return self.remaining()
+
+ def __getitem__(self, key):
+ if type(key) is slice:
+ size = len(self)
+ start, stop, step = slice.indices(size)
+ if start is None: start = self.pos
+ else: start += self.pos
+ if stop is not None: stop += self.pos
+ return self.text[slice(start, stop, step)]
+ elif key < 0:
+ return self.text[key]
+ else:
+ return self.text[self.pos + key]
+
+ def __str__(self):
+ return self.text[self.pos:]
+
+
+def _format_code(code):
+ lines = code.splitlines()
+ format = "%%%dd %%s\n" % len(repr(len(lines) + 1))
+ return "".join([format % (i + 1, line) for (i, line) in enumerate(lines)])
+
+
+def _parse(reader, in_block=None):
+ body = _ChunkList([])
+ while True:
+ # Find next template directive
+ curly = 0
+ while True:
+ curly = reader.find("{", curly)
+ if curly == -1 or curly + 1 == reader.remaining():
+ # EOF
+ if in_block:
+ raise ParseError("Missing {%% end %%} block for %s" %
+ in_block)
+ body.chunks.append(_Text(reader.consume()))
+ return body
+ # If the first curly brace is not the start of a special token,
+ # start searching from the character after it
+ if reader[curly + 1] not in ("{", "%"):
+ curly += 1
+ continue
+ # When there are more than 2 curlies in a row, use the
+ # innermost ones. This is useful when generating languages
+ # like latex where curlies are also meaningful
+ if (curly + 2 < reader.remaining() and
+ reader[curly + 1] == '{' and reader[curly + 2] == '{'):
+ curly += 1
+ continue
+ break
+
+ # Append any text before the special token
+ if curly > 0:
+ body.chunks.append(_Text(reader.consume(curly)))
+
+ start_brace = reader.consume(2)
+ line = reader.line
+
+ # Expression
+ if start_brace == "{{":
+ end = reader.find("}}")
+ if end == -1 or reader.find("\n", 0, end) != -1:
+ raise ParseError("Missing end expression }} on line %d" % line)
+ contents = reader.consume(end).strip()
+ reader.consume(2)
+ if not contents:
+ raise ParseError("Empty expression on line %d" % line)
+ body.chunks.append(_Expression(contents))
+ continue
+
+ # Block
+ assert start_brace == "{%", start_brace
+ end = reader.find("%}")
+ if end == -1 or reader.find("\n", 0, end) != -1:
+ raise ParseError("Missing end block %%} on line %d" % line)
+ contents = reader.consume(end).strip()
+ reader.consume(2)
+ if not contents:
+ raise ParseError("Empty block tag ({%% %%}) on line %d" % line)
+
+ operator, space, suffix = contents.partition(" ")
+ suffix = suffix.strip()
+
+ # Intermediate ("else", "elif", etc) blocks
+ intermediate_blocks = {
+ "else": set(["if", "for", "while"]),
+ "elif": set(["if"]),
+ "except": set(["try"]),
+ "finally": set(["try"]),
+ }
+ allowed_parents = intermediate_blocks.get(operator)
+ if allowed_parents is not None:
+ if not in_block:
+ raise ParseError("%s outside %s block" %
+ (operator, allowed_parents))
+ if in_block not in allowed_parents:
+ raise ParseError("%s block cannot be attached to %s block" % (operator, in_block))
+ body.chunks.append(_IntermediateControlBlock(contents))
+ continue
+
+ # End tag
+ elif operator == "end":
+ if not in_block:
+ raise ParseError("Extra {%% end %%} block on line %d" % line)
+ return body
+
+ elif operator in ("extends", "include", "set", "import", "comment"):
+ if operator == "comment":
+ continue
+ if operator == "extends":
+ suffix = suffix.strip('"').strip("'")
+ if not suffix:
+ raise ParseError("extends missing file path on line %d" % line)
+ block = _ExtendsBlock(suffix)
+ elif operator == "import":
+ if not suffix:
+ raise ParseError("import missing statement on line %d" % line)
+ block = _Statement(contents)
+ elif operator == "include":
+ suffix = suffix.strip('"').strip("'")
+ if not suffix:
+ raise ParseError("include missing file path on line %d" % line)
+ block = _IncludeBlock(suffix, reader)
+ elif operator == "set":
+ if not suffix:
+ raise ParseError("set missing statement on line %d" % line)
+ block = _Statement(suffix)
+ body.chunks.append(block)
+ continue
+
+ elif operator in ("apply", "block", "try", "if", "for", "while"):
+ # parse inner body recursively
+ block_body = _parse(reader, operator)
+ if operator == "apply":
+ if not suffix:
+ raise ParseError("apply missing method name on line %d" % line)
+ block = _ApplyBlock(suffix, block_body)
+ elif operator == "block":
+ if not suffix:
+ raise ParseError("block missing name on line %d" % line)
+ block = _NamedBlock(suffix, block_body)
+ else:
+ block = _ControlBlock(contents, block_body)
+ body.chunks.append(block)
+ continue
+
+ else:
+ raise ParseError("unknown operator: %r" % operator)
--- /dev/null
+Test coverage is almost non-existent, but it's a start. Be sure to
+set PYTHONPATH apprioriately (generally to the root directory of your
+tornado checkout) when running tests to make sure you're getting the
+version of the tornado package that you expect.
\ No newline at end of file
--- /dev/null
+#!/usr/bin/env python
+
+import unittest
+import time
+
+from tornado import ioloop
+
+
+class TestIOLoop(unittest.TestCase):
+ def setUp(self):
+ self.loop = ioloop.IOLoop()
+
+ def tearDown(self):
+ pass
+
+ def _callback(self):
+ self.called = True
+ self.loop.stop()
+
+ def _schedule_callback(self):
+ self.loop.add_callback(self._callback)
+ # Scroll away the time so we can check if we woke up immediately
+ self._start_time = time.time()
+ self.called = False
+
+ def test_add_callback(self):
+ self.loop.add_timeout(time.time(), self._schedule_callback)
+ self.loop.start() # Set some long poll timeout so we can check wakeup
+ self.assertAlmostEqual(time.time(), self._start_time, places=2)
+ self.assertTrue(self.called)
+
+
+if __name__ == "__main__":
+ import logging
+
+ logging.basicConfig(level=logging.DEBUG, format='%(asctime)s:%(msecs)03d %(levelname)-8s %(name)-8s %(message)s', datefmt='%H:%M:%S')
+
+ unittest.main()
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""The Tornado web framework.
+
+The Tornado web framework looks a bit like web.py (http://webpy.org/) or
+Google's webapp (http://code.google.com/appengine/docs/python/tools/webapp/),
+but with additional tools and optimizations to take advantage of the
+Tornado non-blocking web server and tools.
+
+Here is the canonical "Hello, world" example app:
+
+ import tornado.httpserver
+ import tornado.ioloop
+ import tornado.web
+
+ class MainHandler(tornado.web.RequestHandler):
+ def get(self):
+ self.write("Hello, world")
+
+ if __name__ == "__main__":
+ application = tornado.web.Application([
+ (r"/", MainHandler),
+ ])
+ http_server = tornado.httpserver.HTTPServer(application)
+ http_server.listen(8888)
+ tornado.ioloop.IOLoop.instance().start()
+
+See the Tornado walkthrough on GitHub for more details and a good
+getting started guide.
+"""
+
+import base64
+import binascii
+import calendar
+import Cookie
+import cStringIO
+import datetime
+import email.utils
+import escape
+import functools
+import gzip
+import hashlib
+import hmac
+import httplib
+import locale
+import logging
+import mimetypes
+import os.path
+import re
+import stat
+import sys
+import template
+import time
+import types
+import urllib
+import urlparse
+import uuid
+
+class RequestHandler(object):
+ """Subclass this class and define get() or post() to make a handler.
+
+ If you want to support more methods than the standard GET/HEAD/POST, you
+ should override the class variable SUPPORTED_METHODS in your
+ RequestHandler class.
+ """
+ SUPPORTED_METHODS = ("GET", "HEAD", "POST", "DELETE", "PUT")
+
+ def __init__(self, application, request, transforms=None):
+ self.application = application
+ self.request = request
+ self._headers_written = False
+ self._finished = False
+ self._auto_finish = True
+ self._transforms = transforms or []
+ self.ui = _O((n, self._ui_method(m)) for n, m in
+ application.ui_methods.iteritems())
+ self.ui["modules"] = _O((n, self._ui_module(n, m)) for n, m in
+ application.ui_modules.iteritems())
+ self.clear()
+ # Check since connection is not available in WSGI
+ if hasattr(self.request, "connection"):
+ self.request.connection.stream.set_close_callback(
+ self.on_connection_close)
+
+ @property
+ def settings(self):
+ return self.application.settings
+
+ def head(self, *args, **kwargs):
+ raise HTTPError(405)
+
+ def get(self, *args, **kwargs):
+ raise HTTPError(405)
+
+ def post(self, *args, **kwargs):
+ raise HTTPError(405)
+
+ def delete(self, *args, **kwargs):
+ raise HTTPError(405)
+
+ def put(self, *args, **kwargs):
+ raise HTTPError(405)
+
+ def prepare(self):
+ """Called before the actual handler method.
+
+ Useful to override in a handler if you want a common bottleneck for
+ all of your requests.
+ """
+ pass
+
+ def on_connection_close(self):
+ """Called in async handlers if the client closed the connection.
+
+ You may override this to clean up resources associated with
+ long-lived connections.
+
+ Note that the select()-based implementation of IOLoop does not detect
+ closed connections and so this method will not be called until
+ you try (and fail) to produce some output. The epoll- and kqueue-
+ based implementations should detect closed connections even while
+ the request is idle.
+ """
+ pass
+
+ def clear(self):
+ """Resets all headers and content for this response."""
+ self._headers = {
+ "Server": "TornadoServer/0.1",
+ "Content-Type": "text/html; charset=UTF-8",
+ }
+ if not self.request.supports_http_1_1():
+ if self.request.headers.get("Connection") == "Keep-Alive":
+ self.set_header("Connection", "Keep-Alive")
+ self._write_buffer = []
+ self._status_code = 200
+
+ def set_status(self, status_code):
+ """Sets the status code for our response."""
+ assert status_code in httplib.responses
+ self._status_code = status_code
+
+ def set_header(self, name, value):
+ """Sets the given response header name and value.
+
+ If a datetime is given, we automatically format it according to the
+ HTTP specification. If the value is not a string, we convert it to
+ a string. All header values are then encoded as UTF-8.
+ """
+ if isinstance(value, datetime.datetime):
+ t = calendar.timegm(value.utctimetuple())
+ value = email.utils.formatdate(t, localtime=False, usegmt=True)
+ elif isinstance(value, int) or isinstance(value, long):
+ value = str(value)
+ else:
+ value = _utf8(value)
+ # If \n is allowed into the header, it is possible to inject
+ # additional headers or split the request. Also cap length to
+ # prevent obviously erroneous values.
+ safe_value = re.sub(r"[\x00-\x1f]", " ", value)[:4000]
+ if safe_value != value:
+ raise ValueError("Unsafe header value %r", value)
+ self._headers[name] = value
+
+ _ARG_DEFAULT = []
+ def get_argument(self, name, default=_ARG_DEFAULT, strip=True):
+ """Returns the value of the argument with the given name.
+
+ If default is not provided, the argument is considered to be
+ required, and we throw an HTTP 404 exception if it is missing.
+
+ If the argument appears in the url more than once, we return the
+ last value.
+
+ The returned value is always unicode.
+ """
+ args = self.get_arguments(name, strip=strip)
+ if not args:
+ if default is self._ARG_DEFAULT:
+ raise HTTPError(404, "Missing argument %s" % name)
+ return default
+ return args[-1]
+
+ def get_arguments(self, name, strip=True):
+ """Returns a list of the arguments with the given name.
+
+ If the argument is not present, returns an empty list.
+
+ The returned values are always unicode.
+ """
+ values = self.request.arguments.get(name, [])
+ # Get rid of any weird control chars
+ values = [re.sub(r"[\x00-\x08\x0e-\x1f]", " ", x) for x in values]
+ values = [_unicode(x) for x in values]
+ if strip:
+ values = [x.strip() for x in values]
+ return values
+
+
+ @property
+ def cookies(self):
+ """A dictionary of Cookie.Morsel objects."""
+ if not hasattr(self, "_cookies"):
+ self._cookies = Cookie.BaseCookie()
+ if "Cookie" in self.request.headers:
+ try:
+ self._cookies.load(self.request.headers["Cookie"])
+ except:
+ self.clear_all_cookies()
+ return self._cookies
+
+ def get_cookie(self, name, default=None):
+ """Gets the value of the cookie with the given name, else default."""
+ if name in self.cookies:
+ return self.cookies[name].value
+ return default
+
+ def set_cookie(self, name, value, domain=None, expires=None, path="/",
+ expires_days=None, **kwargs):
+ """Sets the given cookie name/value with the given options.
+
+ Additional keyword arguments are set on the Cookie.Morsel
+ directly.
+ See http://docs.python.org/library/cookie.html#morsel-objects
+ for available attributes.
+ """
+ name = _utf8(name)
+ value = _utf8(value)
+ if re.search(r"[\x00-\x20]", name + value):
+ # Don't let us accidentally inject bad stuff
+ raise ValueError("Invalid cookie %r: %r" % (name, value))
+ if not hasattr(self, "_new_cookies"):
+ self._new_cookies = []
+ new_cookie = Cookie.BaseCookie()
+ self._new_cookies.append(new_cookie)
+ new_cookie[name] = value
+ if domain:
+ new_cookie[name]["domain"] = domain
+ if expires_days is not None and not expires:
+ expires = datetime.datetime.utcnow() + datetime.timedelta(
+ days=expires_days)
+ if expires:
+ timestamp = calendar.timegm(expires.utctimetuple())
+ new_cookie[name]["expires"] = email.utils.formatdate(
+ timestamp, localtime=False, usegmt=True)
+ if path:
+ new_cookie[name]["path"] = path
+ for k, v in kwargs.iteritems():
+ new_cookie[name][k] = v
+
+ def clear_cookie(self, name, path="/", domain=None):
+ """Deletes the cookie with the given name."""
+ expires = datetime.datetime.utcnow() - datetime.timedelta(days=365)
+ self.set_cookie(name, value="", path=path, expires=expires,
+ domain=domain)
+
+ def clear_all_cookies(self):
+ """Deletes all the cookies the user sent with this request."""
+ for name in self.cookies.iterkeys():
+ self.clear_cookie(name)
+
+ def set_secure_cookie(self, name, value, expires_days=30, **kwargs):
+ """Signs and timestamps a cookie so it cannot be forged.
+
+ You must specify the 'cookie_secret' setting in your Application
+ to use this method. It should be a long, random sequence of bytes
+ to be used as the HMAC secret for the signature.
+
+ To read a cookie set with this method, use get_secure_cookie().
+ """
+ timestamp = str(int(time.time()))
+ value = base64.b64encode(value)
+ signature = self._cookie_signature(name, value, timestamp)
+ value = "|".join([value, timestamp, signature])
+ self.set_cookie(name, value, expires_days=expires_days, **kwargs)
+
+ def get_secure_cookie(self, name, include_name=True, value=None):
+ """Returns the given signed cookie if it validates, or None.
+
+ In older versions of Tornado (0.1 and 0.2), we did not include the
+ name of the cookie in the cookie signature. To read these old-style
+ cookies, pass include_name=False to this method. Otherwise, all
+ attempts to read old-style cookies will fail (and you may log all
+ your users out whose cookies were written with a previous Tornado
+ version).
+ """
+ if value is None: value = self.get_cookie(name)
+ if not value: return None
+ parts = value.split("|")
+ if len(parts) != 3: return None
+ if include_name:
+ signature = self._cookie_signature(name, parts[0], parts[1])
+ else:
+ signature = self._cookie_signature(parts[0], parts[1])
+ if not _time_independent_equals(parts[2], signature):
+ logging.warning("Invalid cookie signature %r", value)
+ return None
+ timestamp = int(parts[1])
+ if timestamp < time.time() - 31 * 86400:
+ logging.warning("Expired cookie %r", value)
+ return None
+ try:
+ return base64.b64decode(parts[0])
+ except:
+ return None
+
+ def _cookie_signature(self, *parts):
+ self.require_setting("cookie_secret", "secure cookies")
+ hash = hmac.new(self.application.settings["cookie_secret"],
+ digestmod=hashlib.sha1)
+ for part in parts: hash.update(part)
+ return hash.hexdigest()
+
+ def redirect(self, url, permanent=False):
+ """Sends a redirect to the given (optionally relative) URL."""
+ if self._headers_written:
+ raise Exception("Cannot redirect after headers have been written")
+ self.set_status(301 if permanent else 302)
+ # Remove whitespace
+ url = re.sub(r"[\x00-\x20]+", "", _utf8(url))
+ self.set_header("Location", urlparse.urljoin(self.request.uri, url))
+ self.finish()
+
+ def write(self, chunk):
+ """Writes the given chunk to the output buffer.
+
+ To write the output to the network, use the flush() method below.
+
+ If the given chunk is a dictionary, we write it as JSON and set
+ the Content-Type of the response to be text/javascript.
+ """
+ assert not self._finished
+ if isinstance(chunk, dict):
+ chunk = escape.json_encode(chunk)
+ self.set_header("Content-Type", "text/javascript; charset=UTF-8")
+ chunk = _utf8(chunk)
+ self._write_buffer.append(chunk)
+
+ def render(self, template_name, **kwargs):
+ """Renders the template with the given arguments as the response."""
+ html = self.render_string(template_name, **kwargs)
+
+ # Insert the additional JS and CSS added by the modules on the page
+ js_embed = []
+ js_files = []
+ css_embed = []
+ css_files = []
+ html_heads = []
+ html_bodies = []
+ for module in getattr(self, "_active_modules", {}).itervalues():
+ embed_part = module.embedded_javascript()
+ if embed_part: js_embed.append(_utf8(embed_part))
+ file_part = module.javascript_files()
+ if file_part:
+ if isinstance(file_part, basestring):
+ js_files.append(file_part)
+ else:
+ js_files.extend(file_part)
+ embed_part = module.embedded_css()
+ if embed_part: css_embed.append(_utf8(embed_part))
+ file_part = module.css_files()
+ if file_part:
+ if isinstance(file_part, basestring):
+ css_files.append(file_part)
+ else:
+ css_files.extend(file_part)
+ head_part = module.html_head()
+ if head_part: html_heads.append(_utf8(head_part))
+ body_part = module.html_body()
+ if body_part: html_bodies.append(_utf8(body_part))
+ if js_files:
+ # Maintain order of JavaScript files given by modules
+ paths = []
+ unique_paths = set()
+ for path in js_files:
+ if not path.startswith("/") and not path.startswith("http:"):
+ path = self.static_url(path)
+ if path not in unique_paths:
+ paths.append(path)
+ unique_paths.add(path)
+ js = ''.join('<script src="' + escape.xhtml_escape(p) +
+ '" type="text/javascript"></script>'
+ for p in paths)
+ sloc = html.rindex('</body>')
+ html = html[:sloc] + js + '\n' + html[sloc:]
+ if js_embed:
+ js = '<script type="text/javascript">\n//<![CDATA[\n' + \
+ '\n'.join(js_embed) + '\n//]]>\n</script>'
+ sloc = html.rindex('</body>')
+ html = html[:sloc] + js + '\n' + html[sloc:]
+ if css_files:
+ paths = set()
+ for path in css_files:
+ if not path.startswith("/") and not path.startswith("http:"):
+ paths.add(self.static_url(path))
+ else:
+ paths.add(path)
+ css = ''.join('<link href="' + escape.xhtml_escape(p) + '" '
+ 'type="text/css" rel="stylesheet"/>'
+ for p in paths)
+ hloc = html.index('</head>')
+ html = html[:hloc] + css + '\n' + html[hloc:]
+ if css_embed:
+ css = '<style type="text/css">\n' + '\n'.join(css_embed) + \
+ '\n</style>'
+ hloc = html.index('</head>')
+ html = html[:hloc] + css + '\n' + html[hloc:]
+ if html_heads:
+ hloc = html.index('</head>')
+ html = html[:hloc] + ''.join(html_heads) + '\n' + html[hloc:]
+ if html_bodies:
+ hloc = html.index('</body>')
+ html = html[:hloc] + ''.join(html_bodies) + '\n' + html[hloc:]
+ self.finish(html)
+
+ def render_string(self, template_name, **kwargs):
+ """Generate the given template with the given arguments.
+
+ We return the generated string. To generate and write a template
+ as a response, use render() above.
+ """
+ # If no template_path is specified, use the path of the calling file
+ template_path = self.get_template_path()
+ if not template_path:
+ frame = sys._getframe(0)
+ web_file = frame.f_code.co_filename
+ while frame.f_code.co_filename == web_file:
+ frame = frame.f_back
+ template_path = os.path.dirname(frame.f_code.co_filename)
+ if not getattr(RequestHandler, "_templates", None):
+ RequestHandler._templates = {}
+ if template_path not in RequestHandler._templates:
+ loader = self.application.settings.get("template_loader") or\
+ template.Loader(template_path)
+ RequestHandler._templates[template_path] = loader
+ t = RequestHandler._templates[template_path].load(template_name)
+ args = dict(
+ handler=self,
+ request=self.request,
+ current_user=self.current_user,
+ locale=self.locale,
+ _=self.locale.translate,
+ static_url=self.static_url,
+ xsrf_form_html=self.xsrf_form_html,
+ reverse_url=self.application.reverse_url
+ )
+ args.update(self.ui)
+ args.update(kwargs)
+ return t.generate(**args)
+
+ def flush(self, include_footers=False):
+ """Flushes the current output buffer to the nextwork."""
+ if self.application._wsgi:
+ raise Exception("WSGI applications do not support flush()")
+
+ chunk = "".join(self._write_buffer)
+ self._write_buffer = []
+ if not self._headers_written:
+ self._headers_written = True
+ for transform in self._transforms:
+ self._headers, chunk = transform.transform_first_chunk(
+ self._headers, chunk, include_footers)
+ headers = self._generate_headers()
+ else:
+ for transform in self._transforms:
+ chunk = transform.transform_chunk(chunk, include_footers)
+ headers = ""
+
+ # Ignore the chunk and only write the headers for HEAD requests
+ if self.request.method == "HEAD":
+ if headers: self.request.write(headers)
+ return
+
+ if headers or chunk:
+ self.request.write(headers + chunk)
+
+ def finish(self, chunk=None):
+ """Finishes this response, ending the HTTP request."""
+ assert not self._finished
+ if chunk is not None: self.write(chunk)
+
+ # Automatically support ETags and add the Content-Length header if
+ # we have not flushed any content yet.
+ if not self._headers_written:
+ if (self._status_code == 200 and self.request.method == "GET" and
+ "Etag" not in self._headers):
+ hasher = hashlib.sha1()
+ for part in self._write_buffer:
+ hasher.update(part)
+ etag = '"%s"' % hasher.hexdigest()
+ inm = self.request.headers.get("If-None-Match")
+ if inm and inm.find(etag) != -1:
+ self._write_buffer = []
+ self.set_status(304)
+ else:
+ self.set_header("Etag", etag)
+ if "Content-Length" not in self._headers:
+ content_length = sum(len(part) for part in self._write_buffer)
+ self.set_header("Content-Length", content_length)
+
+ if hasattr(self.request, "connection"):
+ # Now that the request is finished, clear the callback we
+ # set on the IOStream (which would otherwise prevent the
+ # garbage collection of the RequestHandler when there
+ # are keepalive connections)
+ self.request.connection.stream.set_close_callback(None)
+
+ if not self.application._wsgi:
+ self.flush(include_footers=True)
+ self.request.finish()
+ self._log()
+ self._finished = True
+
+ def send_error(self, status_code=500, **kwargs):
+ """Sends the given HTTP error code to the browser.
+
+ We also send the error HTML for the given error code as returned by
+ get_error_html. Override that method if you want custom error pages
+ for your application.
+ """
+ if self._headers_written:
+ logging.error("Cannot send error response after headers written")
+ if not self._finished:
+ self.finish()
+ return
+ self.clear()
+ self.set_status(status_code)
+ message = self.get_error_html(status_code, **kwargs)
+ self.finish(message)
+
+ def get_error_html(self, status_code, **kwargs):
+ """Override to implement custom error pages.
+
+ If this error was caused by an uncaught exception, the
+ exception object can be found in kwargs e.g. kwargs['exception']
+ """
+ return "<html><title>%(code)d: %(message)s</title>" \
+ "<body>%(code)d: %(message)s</body></html>" % {
+ "code": status_code,
+ "message": httplib.responses[status_code],
+ }
+
+ @property
+ def locale(self):
+ """The local for the current session.
+
+ Determined by either get_user_locale, which you can override to
+ set the locale based on, e.g., a user preference stored in a
+ database, or get_browser_locale, which uses the Accept-Language
+ header.
+ """
+ if not hasattr(self, "_locale"):
+ self._locale = self.get_user_locale()
+ if not self._locale:
+ self._locale = self.get_browser_locale()
+ assert self._locale
+ return self._locale
+
+ def get_user_locale(self):
+ """Override to determine the locale from the authenticated user.
+
+ If None is returned, we use the Accept-Language header.
+ """
+ return None
+
+ def get_browser_locale(self, default="en_US"):
+ """Determines the user's locale from Accept-Language header.
+
+ See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4
+ """
+ if "Accept-Language" in self.request.headers:
+ languages = self.request.headers["Accept-Language"].split(",")
+ locales = []
+ for language in languages:
+ parts = language.strip().split(";")
+ if len(parts) > 1 and parts[1].startswith("q="):
+ try:
+ score = float(parts[1][2:])
+ except (ValueError, TypeError):
+ score = 0.0
+ else:
+ score = 1.0
+ locales.append((parts[0], score))
+ if locales:
+ locales.sort(key=lambda (l, s): s, reverse=True)
+ codes = [l[0] for l in locales]
+ return locale.get(*codes)
+ return locale.get(default)
+
+ @property
+ def current_user(self):
+ """The authenticated user for this request.
+
+ Determined by either get_current_user, which you can override to
+ set the user based on, e.g., a cookie. If that method is not
+ overridden, this method always returns None.
+
+ We lazy-load the current user the first time this method is called
+ and cache the result after that.
+ """
+ if not hasattr(self, "_current_user"):
+ self._current_user = self.get_current_user()
+ return self._current_user
+
+ def get_current_user(self):
+ """Override to determine the current user from, e.g., a cookie."""
+ return None
+
+ def get_login_url(self):
+ """Override to customize the login URL based on the request.
+
+ By default, we use the 'login_url' application setting.
+ """
+ self.require_setting("login_url", "@tornado.web.authenticated")
+ return self.application.settings["login_url"]
+
+ def get_template_path(self):
+ """Override to customize template path for each handler.
+
+ By default, we use the 'template_path' application setting.
+ Return None to load templates relative to the calling file.
+ """
+ return self.application.settings.get("template_path")
+
+ @property
+ def xsrf_token(self):
+ """The XSRF-prevention token for the current user/session.
+
+ To prevent cross-site request forgery, we set an '_xsrf' cookie
+ and include the same '_xsrf' value as an argument with all POST
+ requests. If the two do not match, we reject the form submission
+ as a potential forgery.
+
+ See http://en.wikipedia.org/wiki/Cross-site_request_forgery
+ """
+ if not hasattr(self, "_xsrf_token"):
+ token = self.get_cookie("_xsrf")
+ if not token:
+ token = binascii.b2a_hex(uuid.uuid4().bytes)
+ expires_days = 30 if self.current_user else None
+ self.set_cookie("_xsrf", token, expires_days=expires_days)
+ self._xsrf_token = token
+ return self._xsrf_token
+
+ def check_xsrf_cookie(self):
+ """Verifies that the '_xsrf' cookie matches the '_xsrf' argument.
+
+ To prevent cross-site request forgery, we set an '_xsrf' cookie
+ and include the same '_xsrf' value as an argument with all POST
+ requests. If the two do not match, we reject the form submission
+ as a potential forgery.
+
+ See http://en.wikipedia.org/wiki/Cross-site_request_forgery
+ """
+ if self.request.headers.get("X-Requested-With") == "XMLHttpRequest":
+ return
+ token = self.get_argument("_xsrf", None)
+ if not token:
+ raise HTTPError(403, "'_xsrf' argument missing from POST")
+ if self.xsrf_token != token:
+ raise HTTPError(403, "XSRF cookie does not match POST argument")
+
+ def xsrf_form_html(self):
+ """An HTML <input/> element to be included with all POST forms.
+
+ It defines the _xsrf input value, which we check on all POST
+ requests to prevent cross-site request forgery. If you have set
+ the 'xsrf_cookies' application setting, you must include this
+ HTML within all of your HTML forms.
+
+ See check_xsrf_cookie() above for more information.
+ """
+ return '<input type="hidden" name="_xsrf" value="' + \
+ escape.xhtml_escape(self.xsrf_token) + '"/>'
+
+ def static_url(self, path):
+ """Returns a static URL for the given relative static file path.
+
+ This method requires you set the 'static_path' setting in your
+ application (which specifies the root directory of your static
+ files).
+
+ We append ?v=<signature> to the returned URL, which makes our
+ static file handler set an infinite expiration header on the
+ returned content. The signature is based on the content of the
+ file.
+
+ If this handler has a "include_host" attribute, we include the
+ full host for every static URL, including the "http://". Set
+ this attribute for handlers whose output needs non-relative static
+ path names.
+ """
+ self.require_setting("static_path", "static_url")
+ if not hasattr(RequestHandler, "_static_hashes"):
+ RequestHandler._static_hashes = {}
+ hashes = RequestHandler._static_hashes
+ if path not in hashes:
+ try:
+ f = open(os.path.join(
+ self.application.settings["static_path"], path))
+ hashes[path] = hashlib.md5(f.read()).hexdigest()
+ f.close()
+ except:
+ logging.error("Could not open static file %r", path)
+ hashes[path] = None
+ base = self.request.protocol + "://" + self.request.host \
+ if getattr(self, "include_host", False) else ""
+ static_url_prefix = self.settings.get('static_url_prefix', '/static/')
+ if hashes.get(path):
+ return base + static_url_prefix + path + "?v=" + hashes[path][:5]
+ else:
+ return base + static_url_prefix + path
+
+ def async_callback(self, callback, *args, **kwargs):
+ """Wrap callbacks with this if they are used on asynchronous requests.
+
+ Catches exceptions and properly finishes the request.
+ """
+ if callback is None:
+ return None
+ if args or kwargs:
+ callback = functools.partial(callback, *args, **kwargs)
+ def wrapper(*args, **kwargs):
+ try:
+ return callback(*args, **kwargs)
+ except Exception, e:
+ if self._headers_written:
+ logging.error("Exception after headers written",
+ exc_info=True)
+ else:
+ self._handle_request_exception(e)
+ return wrapper
+
+ def require_setting(self, name, feature="this feature"):
+ """Raises an exception if the given app setting is not defined."""
+ if not self.application.settings.get(name):
+ raise Exception("You must define the '%s' setting in your "
+ "application to use %s" % (name, feature))
+
+ def reverse_url(self, name, *args):
+ return self.application.reverse_url(name, *args)
+
+ def _execute(self, transforms, *args, **kwargs):
+ """Executes this request with the given output transforms."""
+ self._transforms = transforms
+ try:
+ if self.request.method not in self.SUPPORTED_METHODS:
+ raise HTTPError(405)
+ # If XSRF cookies are turned on, reject form submissions without
+ # the proper cookie
+ if self.request.method == "POST" and \
+ self.application.settings.get("xsrf_cookies"):
+ self.check_xsrf_cookie()
+ self.prepare()
+ if not self._finished:
+ getattr(self, self.request.method.lower())(*args, **kwargs)
+ if self._auto_finish and not self._finished:
+ self.finish()
+ except Exception, e:
+ self._handle_request_exception(e)
+
+ def _generate_headers(self):
+ lines = [self.request.version + " " + str(self._status_code) + " " +
+ httplib.responses[self._status_code]]
+ lines.extend(["%s: %s" % (n, v) for n, v in self._headers.iteritems()])
+ for cookie_dict in getattr(self, "_new_cookies", []):
+ for cookie in cookie_dict.values():
+ lines.append("Set-Cookie: " + cookie.OutputString(None))
+ return "\r\n".join(lines) + "\r\n\r\n"
+
+ def _log(self):
+ if self._status_code < 400:
+ log_method = logging.info
+ elif self._status_code < 500:
+ log_method = logging.warning
+ else:
+ log_method = logging.error
+ request_time = 1000.0 * self.request.request_time()
+ log_method("%d %s %.2fms", self._status_code,
+ self._request_summary(), request_time)
+
+ def _request_summary(self):
+ return self.request.method + " " + self.request.uri + " (" + \
+ self.request.remote_ip + ")"
+
+ def _handle_request_exception(self, e):
+ if isinstance(e, HTTPError):
+ if e.log_message:
+ format = "%d %s: " + e.log_message
+ args = [e.status_code, self._request_summary()] + list(e.args)
+ logging.warning(format, *args)
+ if e.status_code not in httplib.responses:
+ logging.error("Bad HTTP status code: %d", e.status_code)
+ self.send_error(500, exception=e)
+ else:
+ self.send_error(e.status_code, exception=e)
+ else:
+ logging.error("Uncaught exception %s\n%r", self._request_summary(),
+ self.request, exc_info=e)
+ self.send_error(500, exception=e)
+
+ def _ui_module(self, name, module):
+ def render(*args, **kwargs):
+ if not hasattr(self, "_active_modules"):
+ self._active_modules = {}
+ if name not in self._active_modules:
+ self._active_modules[name] = module(self)
+ rendered = self._active_modules[name].render(*args, **kwargs)
+ return rendered
+ return render
+
+ def _ui_method(self, method):
+ return lambda *args, **kwargs: method(self, *args, **kwargs)
+
+
+def asynchronous(method):
+ """Wrap request handler methods with this if they are asynchronous.
+
+ If this decorator is given, the response is not finished when the
+ method returns. It is up to the request handler to call self.finish()
+ to finish the HTTP request. Without this decorator, the request is
+ automatically finished when the get() or post() method returns.
+
+ class MyRequestHandler(web.RequestHandler):
+ @web.asynchronous
+ def get(self):
+ http = httpclient.AsyncHTTPClient()
+ http.fetch("http://friendfeed.com/", self._on_download)
+
+ def _on_download(self, response):
+ self.write("Downloaded!")
+ self.finish()
+
+ """
+ @functools.wraps(method)
+ def wrapper(self, *args, **kwargs):
+ if self.application._wsgi:
+ raise Exception("@asynchronous is not supported for WSGI apps")
+ self._auto_finish = False
+ return method(self, *args, **kwargs)
+ return wrapper
+
+
+def removeslash(method):
+ """Use this decorator to remove trailing slashes from the request path.
+
+ For example, a request to '/foo/' would redirect to '/foo' with this
+ decorator. Your request handler mapping should use a regular expression
+ like r'/foo/*' in conjunction with using the decorator.
+ """
+ @functools.wraps(method)
+ def wrapper(self, *args, **kwargs):
+ if self.request.path.endswith("/"):
+ if self.request.method == "GET":
+ uri = self.request.path.rstrip("/")
+ if self.request.query: uri += "?" + self.request.query
+ self.redirect(uri)
+ return
+ raise HTTPError(404)
+ return method(self, *args, **kwargs)
+ return wrapper
+
+
+def addslash(method):
+ """Use this decorator to add a missing trailing slash to the request path.
+
+ For example, a request to '/foo' would redirect to '/foo/' with this
+ decorator. Your request handler mapping should use a regular expression
+ like r'/foo/?' in conjunction with using the decorator.
+ """
+ @functools.wraps(method)
+ def wrapper(self, *args, **kwargs):
+ if not self.request.path.endswith("/"):
+ if self.request.method == "GET":
+ uri = self.request.path + "/"
+ if self.request.query: uri += "?" + self.request.query
+ self.redirect(uri)
+ return
+ raise HTTPError(404)
+ return method(self, *args, **kwargs)
+ return wrapper
+
+
+class Application(object):
+ """A collection of request handlers that make up a web application.
+
+ Instances of this class are callable and can be passed directly to
+ HTTPServer to serve the application:
+
+ application = web.Application([
+ (r"/", MainPageHandler),
+ ])
+ http_server = httpserver.HTTPServer(application)
+ http_server.listen(8080)
+ ioloop.IOLoop.instance().start()
+
+ The constructor for this class takes in a list of URLSpec objects
+ or (regexp, request_class) tuples. When we receive requests, we
+ iterate over the list in order and instantiate an instance of the
+ first request class whose regexp matches the request path.
+
+ Each tuple can contain an optional third element, which should be a
+ dictionary if it is present. That dictionary is passed as keyword
+ arguments to the contructor of the handler. This pattern is used
+ for the StaticFileHandler below:
+
+ application = web.Application([
+ (r"/static/(.*)", web.StaticFileHandler, {"path": "/var/www"}),
+ ])
+
+ We support virtual hosts with the add_handlers method, which takes in
+ a host regular expression as the first argument:
+
+ application.add_handlers(r"www\.myhost\.com", [
+ (r"/article/([0-9]+)", ArticleHandler),
+ ])
+
+ You can serve static files by sending the static_path setting as a
+ keyword argument. We will serve those files from the /static/ URI
+ (this is configurable with the static_url_prefix setting),
+ and we will serve /favicon.ico and /robots.txt from the same directory.
+ """
+ def __init__(self, handlers=None, default_host="", transforms=None,
+ wsgi=False, **settings):
+ if transforms is None:
+ self.transforms = []
+ if settings.get("gzip"):
+ self.transforms.append(GZipContentEncoding)
+ self.transforms.append(ChunkedTransferEncoding)
+ else:
+ self.transforms = transforms
+ self.handlers = []
+ self.named_handlers = {}
+ self.default_host = default_host
+ self.settings = settings
+ self.ui_modules = {}
+ self.ui_methods = {}
+ self._wsgi = wsgi
+ self._load_ui_modules(settings.get("ui_modules", {}))
+ self._load_ui_methods(settings.get("ui_methods", {}))
+ if self.settings.get("static_path"):
+ path = self.settings["static_path"]
+ handlers = list(handlers or [])
+ static_url_prefix = settings.get("static_url_prefix",
+ "/static/")
+ handlers = [
+ (re.escape(static_url_prefix) + r"(.*)", StaticFileHandler,
+ dict(path=path)),
+ (r"/(favicon\.ico)", StaticFileHandler, dict(path=path)),
+ (r"/(robots\.txt)", StaticFileHandler, dict(path=path)),
+ ] + handlers
+ if handlers: self.add_handlers(".*$", handlers)
+
+ # Automatically reload modified modules
+ if self.settings.get("debug") and not wsgi:
+ import autoreload
+ autoreload.start()
+
+ def add_handlers(self, host_pattern, host_handlers):
+ """Appends the given handlers to our handler list."""
+ if not host_pattern.endswith("$"):
+ host_pattern += "$"
+ handlers = []
+ # The handlers with the wildcard host_pattern are a special
+ # case - they're added in the constructor but should have lower
+ # precedence than the more-precise handlers added later.
+ # If a wildcard handler group exists, it should always be last
+ # in the list, so insert new groups just before it.
+ if self.handlers and self.handlers[-1][0].pattern == '.*$':
+ self.handlers.insert(-1, (re.compile(host_pattern), handlers))
+ else:
+ self.handlers.append((re.compile(host_pattern), handlers))
+
+ for spec in host_handlers:
+ if type(spec) is type(()):
+ assert len(spec) in (2, 3)
+ pattern = spec[0]
+ handler = spec[1]
+ if len(spec) == 3:
+ kwargs = spec[2]
+ else:
+ kwargs = {}
+ spec = URLSpec(pattern, handler, kwargs)
+ handlers.append(spec)
+ if spec.name:
+ if spec.name in self.named_handlers:
+ logging.warning(
+ "Multiple handlers named %s; replacing previous value",
+ spec.name)
+ self.named_handlers[spec.name] = spec
+
+ def add_transform(self, transform_class):
+ """Adds the given OutputTransform to our transform list."""
+ self.transforms.append(transform_class)
+
+ def _get_host_handlers(self, request):
+ host = request.host.lower().split(':')[0]
+ for pattern, handlers in self.handlers:
+ if pattern.match(host):
+ return handlers
+ # Look for default host if not behind load balancer (for debugging)
+ if "X-Real-Ip" not in request.headers:
+ for pattern, handlers in self.handlers:
+ if pattern.match(self.default_host):
+ return handlers
+ return None
+
+ def _load_ui_methods(self, methods):
+ if type(methods) is types.ModuleType:
+ self._load_ui_methods(dict((n, getattr(methods, n))
+ for n in dir(methods)))
+ elif isinstance(methods, list):
+ for m in methods: self._load_ui_methods(m)
+ else:
+ for name, fn in methods.iteritems():
+ if not name.startswith("_") and hasattr(fn, "__call__") \
+ and name[0].lower() == name[0]:
+ self.ui_methods[name] = fn
+
+ def _load_ui_modules(self, modules):
+ if type(modules) is types.ModuleType:
+ self._load_ui_modules(dict((n, getattr(modules, n))
+ for n in dir(modules)))
+ elif isinstance(modules, list):
+ for m in modules: self._load_ui_modules(m)
+ else:
+ assert isinstance(modules, dict)
+ for name, cls in modules.iteritems():
+ try:
+ if issubclass(cls, UIModule):
+ self.ui_modules[name] = cls
+ except TypeError:
+ pass
+
+ def __call__(self, request):
+ """Called by HTTPServer to execute the request."""
+ transforms = [t(request) for t in self.transforms]
+ handler = None
+ args = []
+ kwargs = {}
+ handlers = self._get_host_handlers(request)
+ if not handlers:
+ handler = RedirectHandler(
+ request, "http://" + self.default_host + "/")
+ else:
+ for spec in handlers:
+ match = spec.regex.match(request.path)
+ if match:
+ handler = spec.handler_class(self, request, **spec.kwargs)
+ # Pass matched groups to the handler. Since
+ # match.groups() includes both named and unnamed groups,
+ # we want to use either groups or groupdict but not both.
+ kwargs = dict((k, urllib.unquote(v))
+ for (k, v) in match.groupdict().iteritems())
+ if kwargs:
+ args = []
+ else:
+ args = [urllib.unquote(s) for s in match.groups()]
+ break
+ if not handler:
+ handler = ErrorHandler(self, request, 404)
+
+ # In debug mode, re-compile templates and reload static files on every
+ # request so you don't need to restart to see changes
+ if self.settings.get("debug"):
+ if getattr(RequestHandler, "_templates", None):
+ map(lambda loader: loader.reset(),
+ RequestHandler._templates.values())
+ RequestHandler._static_hashes = {}
+
+ handler._execute(transforms, *args, **kwargs)
+ return handler
+
+ def reverse_url(self, name, *args):
+ """Returns a URL path for handler named `name`
+
+ The handler must be added to the application as a named URLSpec
+ """
+ if name in self.named_handlers:
+ return self.named_handlers[name].reverse(*args)
+ raise KeyError("%s not found in named urls" % name)
+
+
+class HTTPError(Exception):
+ """An exception that will turn into an HTTP error response."""
+ def __init__(self, status_code, log_message=None, *args):
+ self.status_code = status_code
+ self.log_message = log_message
+ self.args = args
+
+ def __str__(self):
+ message = "HTTP %d: %s" % (
+ self.status_code, httplib.responses[self.status_code])
+ if self.log_message:
+ return message + " (" + (self.log_message % self.args) + ")"
+ else:
+ return message
+
+
+class ErrorHandler(RequestHandler):
+ """Generates an error response with status_code for all requests."""
+ def __init__(self, application, request, status_code):
+ RequestHandler.__init__(self, application, request)
+ self.set_status(status_code)
+
+ def prepare(self):
+ raise HTTPError(self._status_code)
+
+
+class RedirectHandler(RequestHandler):
+ """Redirects the client to the given URL for all GET requests.
+
+ You should provide the keyword argument "url" to the handler, e.g.:
+
+ application = web.Application([
+ (r"/oldpath", web.RedirectHandler, {"url": "/newpath"}),
+ ])
+ """
+ def __init__(self, application, request, url, permanent=True):
+ RequestHandler.__init__(self, application, request)
+ self._url = url
+ self._permanent = permanent
+
+ def get(self):
+ self.redirect(self._url, permanent=self._permanent)
+
+
+class StaticFileHandler(RequestHandler):
+ """A simple handler that can serve static content from a directory.
+
+ To map a path to this handler for a static data directory /var/www,
+ you would add a line to your application like:
+
+ application = web.Application([
+ (r"/static/(.*)", web.StaticFileHandler, {"path": "/var/www"}),
+ ])
+
+ The local root directory of the content should be passed as the "path"
+ argument to the handler.
+
+ To support aggressive browser caching, if the argument "v" is given
+ with the path, we set an infinite HTTP expiration header. So, if you
+ want browsers to cache a file indefinitely, send them to, e.g.,
+ /static/images/myimage.png?v=xxx.
+ """
+ def __init__(self, application, request, path):
+ RequestHandler.__init__(self, application, request)
+ self.root = os.path.abspath(path) + os.path.sep
+
+ def head(self, path):
+ self.get(path, include_body=False)
+
+ def get(self, path, include_body=True):
+ abspath = os.path.abspath(os.path.join(self.root, path))
+ if not abspath.startswith(self.root):
+ raise HTTPError(403, "%s is not in root static directory", path)
+ if not os.path.exists(abspath):
+ raise HTTPError(404)
+ if not os.path.isfile(abspath):
+ raise HTTPError(403, "%s is not a file", path)
+
+ stat_result = os.stat(abspath)
+ modified = datetime.datetime.fromtimestamp(stat_result[stat.ST_MTIME])
+
+ self.set_header("Last-Modified", modified)
+ if "v" in self.request.arguments:
+ self.set_header("Expires", datetime.datetime.utcnow() + \
+ datetime.timedelta(days=365*10))
+ self.set_header("Cache-Control", "max-age=" + str(86400*365*10))
+ else:
+ self.set_header("Cache-Control", "public")
+ mime_type, encoding = mimetypes.guess_type(abspath)
+ if mime_type:
+ self.set_header("Content-Type", mime_type)
+
+ self.set_extra_headers(path)
+
+ # Check the If-Modified-Since, and don't send the result if the
+ # content has not been modified
+ ims_value = self.request.headers.get("If-Modified-Since")
+ if ims_value is not None:
+ date_tuple = email.utils.parsedate(ims_value)
+ if_since = datetime.datetime.fromtimestamp(time.mktime(date_tuple))
+ if if_since >= modified:
+ self.set_status(304)
+ return
+
+ if not include_body:
+ return
+ self.set_header("Content-Length", stat_result[stat.ST_SIZE])
+ file = open(abspath, "rb")
+ try:
+ self.write(file.read())
+ finally:
+ file.close()
+
+ def set_extra_headers(self, path):
+ """For subclass to add extra headers to the response"""
+ pass
+
+
+class FallbackHandler(RequestHandler):
+ """A RequestHandler that wraps another HTTP server callback.
+
+ The fallback is a callable object that accepts an HTTPRequest,
+ such as an Application or tornado.wsgi.WSGIContainer. This is most
+ useful to use both tornado RequestHandlers and WSGI in the same server.
+ Typical usage:
+ wsgi_app = tornado.wsgi.WSGIContainer(
+ django.core.handlers.wsgi.WSGIHandler())
+ application = tornado.web.Application([
+ (r"/foo", FooHandler),
+ (r".*", FallbackHandler, dict(fallback=wsgi_app),
+ ])
+ """
+ def __init__(self, app, request, fallback):
+ RequestHandler.__init__(self, app, request)
+ self.fallback = fallback
+
+ def prepare(self):
+ self.fallback(self.request)
+ self._finished = True
+
+
+class OutputTransform(object):
+ """A transform modifies the result of an HTTP request (e.g., GZip encoding)
+
+ A new transform instance is created for every request. See the
+ ChunkedTransferEncoding example below if you want to implement a
+ new Transform.
+ """
+ def __init__(self, request):
+ pass
+
+ def transform_first_chunk(self, headers, chunk, finishing):
+ return headers, chunk
+
+ def transform_chunk(self, chunk, finishing):
+ return chunk
+
+
+class GZipContentEncoding(OutputTransform):
+ """Applies the gzip content encoding to the response.
+
+ See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11
+ """
+ CONTENT_TYPES = set([
+ "text/plain", "text/html", "text/css", "text/xml",
+ "application/x-javascript", "application/xml", "application/atom+xml",
+ "text/javascript", "application/json", "application/xhtml+xml"])
+ MIN_LENGTH = 5
+
+ def __init__(self, request):
+ self._gzipping = request.supports_http_1_1() and \
+ "gzip" in request.headers.get("Accept-Encoding", "")
+
+ def transform_first_chunk(self, headers, chunk, finishing):
+ if self._gzipping:
+ ctype = headers.get("Content-Type", "").split(";")[0]
+ self._gzipping = (ctype in self.CONTENT_TYPES) and \
+ (not finishing or len(chunk) >= self.MIN_LENGTH) and \
+ (finishing or "Content-Length" not in headers) and \
+ ("Content-Encoding" not in headers)
+ if self._gzipping:
+ headers["Content-Encoding"] = "gzip"
+ self._gzip_value = cStringIO.StringIO()
+ self._gzip_file = gzip.GzipFile(mode="w", fileobj=self._gzip_value)
+ self._gzip_pos = 0
+ chunk = self.transform_chunk(chunk, finishing)
+ if "Content-Length" in headers:
+ headers["Content-Length"] = str(len(chunk))
+ return headers, chunk
+
+ def transform_chunk(self, chunk, finishing):
+ if self._gzipping:
+ self._gzip_file.write(chunk)
+ if finishing:
+ self._gzip_file.close()
+ else:
+ self._gzip_file.flush()
+ chunk = self._gzip_value.getvalue()
+ if self._gzip_pos > 0:
+ chunk = chunk[self._gzip_pos:]
+ self._gzip_pos += len(chunk)
+ return chunk
+
+
+class ChunkedTransferEncoding(OutputTransform):
+ """Applies the chunked transfer encoding to the response.
+
+ See http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1
+ """
+ def __init__(self, request):
+ self._chunking = request.supports_http_1_1()
+
+ def transform_first_chunk(self, headers, chunk, finishing):
+ if self._chunking:
+ # No need to chunk the output if a Content-Length is specified
+ if "Content-Length" in headers or "Transfer-Encoding" in headers:
+ self._chunking = False
+ else:
+ headers["Transfer-Encoding"] = "chunked"
+ chunk = self.transform_chunk(chunk, finishing)
+ return headers, chunk
+
+ def transform_chunk(self, block, finishing):
+ if self._chunking:
+ # Don't write out empty chunks because that means END-OF-STREAM
+ # with chunked encoding
+ if block:
+ block = ("%x" % len(block)) + "\r\n" + block + "\r\n"
+ if finishing:
+ block += "0\r\n\r\n"
+ return block
+
+
+def authenticated(method):
+ """Decorate methods with this to require that the user be logged in."""
+ @functools.wraps(method)
+ def wrapper(self, *args, **kwargs):
+ if not self.current_user:
+ if self.request.method == "GET":
+ url = self.get_login_url()
+ if "?" not in url:
+ url += "?" + urllib.urlencode(dict(next=self.request.uri))
+ self.redirect(url)
+ return
+ raise HTTPError(403)
+ return method(self, *args, **kwargs)
+ return wrapper
+
+
+class UIModule(object):
+ """A UI re-usable, modular unit on a page.
+
+ UI modules often execute additional queries, and they can include
+ additional CSS and JavaScript that will be included in the output
+ page, which is automatically inserted on page render.
+ """
+ def __init__(self, handler):
+ self.handler = handler
+ self.request = handler.request
+ self.ui = handler.ui
+ self.current_user = handler.current_user
+ self.locale = handler.locale
+
+ def render(self, *args, **kwargs):
+ raise NotImplementedError()
+
+ def embedded_javascript(self):
+ """Returns a JavaScript string that will be embedded in the page."""
+ return None
+
+ def javascript_files(self):
+ """Returns a list of JavaScript files required by this module."""
+ return None
+
+ def embedded_css(self):
+ """Returns a CSS string that will be embedded in the page."""
+ return None
+
+ def css_files(self):
+ """Returns a list of CSS files required by this module."""
+ return None
+
+ def html_head(self):
+ """Returns a CSS string that will be put in the <head/> element"""
+ return None
+
+ def html_body(self):
+ """Returns an HTML string that will be put in the <body/> element"""
+ return None
+
+ def render_string(self, path, **kwargs):
+ return self.handler.render_string(path, **kwargs)
+
+class URLSpec(object):
+ """Specifies mappings between URLs and handlers."""
+ def __init__(self, pattern, handler_class, kwargs={}, name=None):
+ """Creates a URLSpec.
+
+ Parameters:
+ pattern: Regular expression to be matched. Any groups in the regex
+ will be passed in to the handler's get/post/etc methods as
+ arguments.
+ handler_class: RequestHandler subclass to be invoked.
+ kwargs (optional): A dictionary of additional arguments to be passed
+ to the handler's constructor.
+ name (optional): A name for this handler. Used by
+ Application.reverse_url.
+ """
+ if not pattern.endswith('$'):
+ pattern += '$'
+ self.regex = re.compile(pattern)
+ self.handler_class = handler_class
+ self.kwargs = kwargs
+ self.name = name
+ self._path, self._group_count = self._find_groups()
+
+ def _find_groups(self):
+ """Returns a tuple (reverse string, group count) for a url.
+
+ For example: Given the url pattern /([0-9]{4})/([a-z-]+)/, this method
+ would return ('/%s/%s/', 2).
+ """
+ pattern = self.regex.pattern
+ if pattern.startswith('^'):
+ pattern = pattern[1:]
+ if pattern.endswith('$'):
+ pattern = pattern[:-1]
+
+ if self.regex.groups != pattern.count('('):
+ # The pattern is too complicated for our simplistic matching,
+ # so we can't support reversing it.
+ return (None, None)
+
+ pieces = []
+ for fragment in pattern.split('('):
+ if ')' in fragment:
+ paren_loc = fragment.index(')')
+ if paren_loc >= 0:
+ pieces.append('%s' + fragment[paren_loc + 1:])
+ else:
+ pieces.append(fragment)
+
+ return (''.join(pieces), self.regex.groups)
+
+ def reverse(self, *args):
+ assert self._path is not None, \
+ "Cannot reverse url regex " + self.regex.pattern
+ assert len(args) == self._group_count, "required number of arguments "\
+ "not found"
+ if not len(args):
+ return self._path
+ return self._path % tuple([str(a) for a in args])
+
+url = URLSpec
+
+def _utf8(s):
+ if isinstance(s, unicode):
+ return s.encode("utf-8")
+ assert isinstance(s, str)
+ return s
+
+
+def _unicode(s):
+ if isinstance(s, str):
+ try:
+ return s.decode("utf-8")
+ except UnicodeDecodeError:
+ raise HTTPError(400, "Non-utf8 argument")
+ assert isinstance(s, unicode)
+ return s
+
+
+def _time_independent_equals(a, b):
+ if len(a) != len(b):
+ return False
+ result = 0
+ for x, y in zip(a, b):
+ result |= ord(x) ^ ord(y)
+ return result == 0
+
+
+class _O(dict):
+ """Makes a dictionary behave like an object."""
+ def __getattr__(self, name):
+ try:
+ return self[name]
+ except KeyError:
+ raise AttributeError(name)
+
+ def __setattr__(self, name, value):
+ self[name] = value
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import functools
+import logging
+import tornado.escape
+import tornado.web
+
+class WebSocketHandler(tornado.web.RequestHandler):
+ """A request handler for HTML 5 Web Sockets.
+
+ See http://www.w3.org/TR/2009/WD-websockets-20091222/ for details on the
+ JavaScript interface. We implement the protocol as specified at
+ http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-55.
+
+ Here is an example Web Socket handler that echos back all received messages
+ back to the client:
+
+ class EchoWebSocket(websocket.WebSocketHandler):
+ def open(self):
+ self.receive_message(self.on_message)
+
+ def on_message(self, message):
+ self.write_message(u"You said: " + message)
+ # receive_message only reads a single message, so call it
+ # again to listen for the next one
+ self.receive_message(self.on_message)
+
+ Web Sockets are not standard HTTP connections. The "handshake" is HTTP,
+ but after the handshake, the protocol is message-based. Consequently,
+ most of the Tornado HTTP facilities are not available in handlers of this
+ type. The only communication methods available to you are send_message()
+ and receive_message(). Likewise, your request handler class should
+ implement open() method rather than get() or post().
+
+ If you map the handler above to "/websocket" in your application, you can
+ invoke it in JavaScript with:
+
+ var ws = new WebSocket("ws://localhost:8888/websocket");
+ ws.onopen = function() {
+ ws.send("Hello, world");
+ };
+ ws.onmessage = function (evt) {
+ alert(evt.data);
+ };
+
+ This script pops up an alert box that says "You said: Hello, world".
+ """
+ def __init__(self, application, request):
+ tornado.web.RequestHandler.__init__(self, application, request)
+ self.stream = request.connection.stream
+
+ def _execute(self, transforms, *args, **kwargs):
+ if self.request.headers.get("Upgrade") != "WebSocket" or \
+ self.request.headers.get("Connection") != "Upgrade" or \
+ not self.request.headers.get("Origin"):
+ message = "Expected WebSocket headers"
+ self.stream.write(
+ "HTTP/1.1 403 Forbidden\r\nContent-Length: " +
+ str(len(message)) + "\r\n\r\n" + message)
+ return
+ self.stream.write(
+ "HTTP/1.1 101 Web Socket Protocol Handshake\r\n"
+ "Upgrade: WebSocket\r\n"
+ "Connection: Upgrade\r\n"
+ "Server: TornadoServer/0.1\r\n"
+ "WebSocket-Origin: " + self.request.headers["Origin"] + "\r\n"
+ "WebSocket-Location: ws://" + self.request.host +
+ self.request.path + "\r\n\r\n")
+ self.async_callback(self.open)(*args, **kwargs)
+
+ def write_message(self, message):
+ """Sends the given message to the client of this Web Socket."""
+ if isinstance(message, dict):
+ message = tornado.escape.json_encode(message)
+ if isinstance(message, unicode):
+ message = message.encode("utf-8")
+ assert isinstance(message, str)
+ self.stream.write("\x00" + message + "\xff")
+
+ def receive_message(self, callback):
+ """Calls callback when the browser calls send() on this Web Socket."""
+ callback = self.async_callback(callback)
+ self.stream.read_bytes(
+ 1, functools.partial(self._on_frame_type, callback))
+
+ def close(self):
+ """Closes this Web Socket.
+
+ The browser will receive the onclose event for the open web socket
+ when this method is called.
+ """
+ self.stream.close()
+
+ def async_callback(self, callback, *args, **kwargs):
+ """Wrap callbacks with this if they are used on asynchronous requests.
+
+ Catches exceptions properly and closes this Web Socket if an exception
+ is uncaught.
+ """
+ if args or kwargs:
+ callback = functools.partial(callback, *args, **kwargs)
+ def wrapper(*args, **kwargs):
+ try:
+ return callback(*args, **kwargs)
+ except Exception, e:
+ logging.error("Uncaught exception in %s",
+ self.request.path, exc_info=True)
+ self.stream.close()
+ return wrapper
+
+ def _on_frame_type(self, callback, byte):
+ if ord(byte) & 0x80 == 0x80:
+ raise Exception("Length-encoded format not yet supported")
+ self.stream.read_until(
+ "\xff", functools.partial(self._on_end_delimiter, callback))
+
+ def _on_end_delimiter(self, callback, frame):
+ callback(frame[:-1].decode("utf-8", "replace"))
+
+ def _not_supported(self, *args, **kwargs):
+ raise Exception("Method not supported for Web Sockets")
+
+for method in ["write", "redirect", "set_header", "send_error", "set_cookie",
+ "set_status", "flush", "finish"]:
+ setattr(WebSocketHandler, method, WebSocketHandler._not_supported)
--- /dev/null
+# NOTE: win32 support is currently experimental, and not recommended
+# for production use.
+
+import ctypes
+import ctypes.wintypes
+import os
+import socket
+import errno
+
+
+# See: http://msdn.microsoft.com/en-us/library/ms738573(VS.85).aspx
+ioctlsocket = ctypes.windll.ws2_32.ioctlsocket
+ioctlsocket.argtypes = (ctypes.wintypes.HANDLE, ctypes.wintypes.LONG, ctypes.wintypes.ULONG)
+ioctlsocket.restype = ctypes.c_int
+
+# See: http://msdn.microsoft.com/en-us/library/ms724935(VS.85).aspx
+SetHandleInformation = ctypes.windll.kernel32.SetHandleInformation
+SetHandleInformation.argtypes = (ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)
+SetHandleInformation.restype = ctypes.wintypes.BOOL
+
+HANDLE_FLAG_INHERIT = 0x00000001
+
+
+F_GETFD = 1
+F_SETFD = 2
+F_GETFL = 3
+F_SETFL = 4
+
+FD_CLOEXEC = 1
+
+os.O_NONBLOCK = 2048
+
+FIONBIO = 126
+
+
+def fcntl(fd, op, arg=0):
+ if op == F_GETFD or op == F_GETFL:
+ return 0
+ elif op == F_SETFD:
+ # Check that the flag is CLOEXEC and translate
+ if arg == FD_CLOEXEC:
+ success = SetHandleInformation(fd, HANDLE_FLAG_INHERIT, arg)
+ if not success:
+ raise ctypes.GetLastError()
+ else:
+ raise ValueError("Unsupported arg")
+ #elif op == F_SETFL:
+ ## Check that the flag is NONBLOCK and translate
+ #if arg == os.O_NONBLOCK:
+ ##pass
+ #result = ioctlsocket(fd, FIONBIO, 1)
+ #if result != 0:
+ #raise ctypes.GetLastError()
+ #else:
+ #raise ValueError("Unsupported arg")
+ else:
+ raise ValueError("Unsupported op")
+
+
+class Pipe(object):
+ """Create an OS independent asynchronous pipe"""
+ def __init__(self):
+ # Based on Zope async.py: http://svn.zope.org/zc.ngi/trunk/src/zc/ngi/async.py
+
+ self.writer = socket.socket()
+ # Disable buffering -- pulling the trigger sends 1 byte,
+ # and we want that sent immediately, to wake up ASAP.
+ self.writer.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+
+ count = 0
+ while 1:
+ count += 1
+ # Bind to a local port; for efficiency, let the OS pick
+ # a free port for us.
+ # Unfortunately, stress tests showed that we may not
+ # be able to connect to that port ("Address already in
+ # use") despite that the OS picked it. This appears
+ # to be a race bug in the Windows socket implementation.
+ # So we loop until a connect() succeeds (almost always
+ # on the first try). See the long thread at
+ # http://mail.zope.org/pipermail/zope/2005-July/160433.html
+ # for hideous details.
+ a = socket.socket()
+ a.bind(("127.0.0.1", 0))
+ connect_address = a.getsockname() # assigned (host, port) pair
+ a.listen(1)
+ try:
+ self.writer.connect(connect_address)
+ break # success
+ except socket.error, detail:
+ if detail[0] != errno.WSAEADDRINUSE:
+ # "Address already in use" is the only error
+ # I've seen on two WinXP Pro SP2 boxes, under
+ # Pythons 2.3.5 and 2.4.1.
+ raise
+ # (10048, 'Address already in use')
+ # assert count <= 2 # never triggered in Tim's tests
+ if count >= 10: # I've never seen it go above 2
+ a.close()
+ self.writer.close()
+ raise socket.error("Cannot bind trigger!")
+ # Close `a` and try again. Note: I originally put a short
+ # sleep() here, but it didn't appear to help or hurt.
+ a.close()
+
+ self.reader, addr = a.accept()
+ self.reader.setblocking(0)
+ self.writer.setblocking(0)
+ a.close()
+ self.reader_fd = self.reader.fileno()
+
+ def read(self):
+ """Emulate a file descriptors read method"""
+ try:
+ return self.reader.recv(1)
+ except socket.error, ex:
+ if ex.args[0] == errno.EWOULDBLOCK:
+ raise IOError
+ raise
+
+ def write(self, data):
+ """Emulate a file descriptors write method"""
+ return self.writer.send(data)
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""WSGI support for the Tornado web framework.
+
+We export WSGIApplication, which is very similar to web.Application, except
+no asynchronous methods are supported (since WSGI does not support
+non-blocking requests properly). If you call self.flush() or other
+asynchronous methods in your request handlers running in a WSGIApplication,
+we throw an exception.
+
+Example usage:
+
+ import tornado.web
+ import tornado.wsgi
+ import wsgiref.simple_server
+
+ class MainHandler(tornado.web.RequestHandler):
+ def get(self):
+ self.write("Hello, world")
+
+ if __name__ == "__main__":
+ application = tornado.wsgi.WSGIApplication([
+ (r"/", MainHandler),
+ ])
+ server = wsgiref.simple_server.make_server('', 8888, application)
+ server.serve_forever()
+
+See the 'appengine' demo for an example of using this module to run
+a Tornado app on Google AppEngine.
+
+Since no asynchronous methods are available for WSGI applications, the
+httpclient and auth modules are both not available for WSGI applications.
+
+We also export WSGIContainer, which lets you run other WSGI-compatible
+frameworks on the Tornado HTTP server and I/O loop. See WSGIContainer for
+details and documentation.
+"""
+
+import cgi
+import cStringIO
+import escape
+import httplib
+import httputil
+import logging
+import sys
+import time
+import urllib
+import web
+
+class WSGIApplication(web.Application):
+ """A WSGI-equivalent of web.Application.
+
+ We support the same interface, but handlers running in a WSGIApplication
+ do not support flush() or asynchronous methods.
+ """
+ def __init__(self, handlers=None, default_host="", **settings):
+ web.Application.__init__(self, handlers, default_host, transforms=[],
+ wsgi=True, **settings)
+
+ def __call__(self, environ, start_response):
+ handler = web.Application.__call__(self, HTTPRequest(environ))
+ assert handler._finished
+ status = str(handler._status_code) + " " + \
+ httplib.responses[handler._status_code]
+ headers = handler._headers.items()
+ for cookie_dict in getattr(handler, "_new_cookies", []):
+ for cookie in cookie_dict.values():
+ headers.append(("Set-Cookie", cookie.OutputString(None)))
+ start_response(status, headers)
+ return handler._write_buffer
+
+
+class HTTPRequest(object):
+ """Mimics httpserver.HTTPRequest for WSGI applications."""
+ def __init__(self, environ):
+ """Parses the given WSGI environ to construct the request."""
+ self.method = environ["REQUEST_METHOD"]
+ self.path = urllib.quote(environ.get("SCRIPT_NAME", ""))
+ self.path += urllib.quote(environ.get("PATH_INFO", ""))
+ self.uri = self.path
+ self.arguments = {}
+ self.query = environ.get("QUERY_STRING", "")
+ if self.query:
+ self.uri += "?" + self.query
+ arguments = cgi.parse_qs(self.query)
+ for name, values in arguments.iteritems():
+ values = [v for v in values if v]
+ if values: self.arguments[name] = values
+ self.version = "HTTP/1.1"
+ self.headers = httputil.HTTPHeaders()
+ if environ.get("CONTENT_TYPE"):
+ self.headers["Content-Type"] = environ["CONTENT_TYPE"]
+ if environ.get("CONTENT_LENGTH"):
+ self.headers["Content-Length"] = int(environ["CONTENT_LENGTH"])
+ for key in environ:
+ if key.startswith("HTTP_"):
+ self.headers[key[5:].replace("_", "-")] = environ[key]
+ if self.headers.get("Content-Length"):
+ self.body = environ["wsgi.input"].read()
+ else:
+ self.body = ""
+ self.protocol = environ["wsgi.url_scheme"]
+ self.remote_ip = environ.get("REMOTE_ADDR", "")
+ if environ.get("HTTP_HOST"):
+ self.host = environ["HTTP_HOST"]
+ else:
+ self.host = environ["SERVER_NAME"]
+
+ # Parse request body
+ self.files = {}
+ content_type = self.headers.get("Content-Type", "")
+ if content_type.startswith("application/x-www-form-urlencoded"):
+ for name, values in cgi.parse_qs(self.body).iteritems():
+ self.arguments.setdefault(name, []).extend(values)
+ elif content_type.startswith("multipart/form-data"):
+ if 'boundary=' in content_type:
+ boundary = content_type.split('boundary=',1)[1]
+ if boundary: self._parse_mime_body(boundary)
+ else:
+ logging.warning("Invalid multipart/form-data")
+
+ self._start_time = time.time()
+ self._finish_time = None
+
+ def supports_http_1_1(self):
+ """Returns True if this request supports HTTP/1.1 semantics"""
+ return self.version == "HTTP/1.1"
+
+ def full_url(self):
+ """Reconstructs the full URL for this request."""
+ return self.protocol + "://" + self.host + self.uri
+
+ def request_time(self):
+ """Returns the amount of time it took for this request to execute."""
+ if self._finish_time is None:
+ return time.time() - self._start_time
+ else:
+ return self._finish_time - self._start_time
+
+ def _parse_mime_body(self, boundary):
+ if boundary.startswith('"') and boundary.endswith('"'):
+ boundary = boundary[1:-1]
+ if self.body.endswith("\r\n"):
+ footer_length = len(boundary) + 6
+ else:
+ footer_length = len(boundary) + 4
+ parts = self.body[:-footer_length].split("--" + boundary + "\r\n")
+ for part in parts:
+ if not part: continue
+ eoh = part.find("\r\n\r\n")
+ if eoh == -1:
+ logging.warning("multipart/form-data missing headers")
+ continue
+ headers = httputil.HTTPHeaders.parse(part[:eoh])
+ name_header = headers.get("Content-Disposition", "")
+ if not name_header.startswith("form-data;") or \
+ not part.endswith("\r\n"):
+ logging.warning("Invalid multipart/form-data")
+ continue
+ value = part[eoh + 4:-2]
+ name_values = {}
+ for name_part in name_header[10:].split(";"):
+ name, name_value = name_part.strip().split("=", 1)
+ name_values[name] = name_value.strip('"').decode("utf-8")
+ if not name_values.get("name"):
+ logging.warning("multipart/form-data value missing name")
+ continue
+ name = name_values["name"]
+ if name_values.get("filename"):
+ ctype = headers.get("Content-Type", "application/unknown")
+ self.files.setdefault(name, []).append(dict(
+ filename=name_values["filename"], body=value,
+ content_type=ctype))
+ else:
+ self.arguments.setdefault(name, []).append(value)
+
+
+class WSGIContainer(object):
+ """Makes a WSGI-compatible function runnable on Tornado's HTTP server.
+
+ Wrap a WSGI function in a WSGIContainer and pass it to HTTPServer to
+ run it. For example:
+
+ def simple_app(environ, start_response):
+ status = "200 OK"
+ response_headers = [("Content-type", "text/plain")]
+ start_response(status, response_headers)
+ return ["Hello world!\n"]
+
+ container = tornado.wsgi.WSGIContainer(simple_app)
+ http_server = tornado.httpserver.HTTPServer(container)
+ http_server.listen(8888)
+ tornado.ioloop.IOLoop.instance().start()
+
+ This class is intended to let other frameworks (Django, web.py, etc)
+ run on the Tornado HTTP server and I/O loop. It has not yet been
+ thoroughly tested in production.
+ """
+ def __init__(self, wsgi_application):
+ self.wsgi_application = wsgi_application
+
+ def __call__(self, request):
+ data = {}
+ response = []
+ def start_response(status, response_headers, exc_info=None):
+ data["status"] = status
+ data["headers"] = response_headers
+ return response.append
+ app_response = self.wsgi_application(
+ WSGIContainer.environ(request), start_response)
+ response.extend(app_response)
+ body = "".join(response)
+ if hasattr(app_response, "close"):
+ app_response.close()
+ if not data: raise Exception("WSGI app did not call start_response")
+
+ status_code = int(data["status"].split()[0])
+ headers = data["headers"]
+ header_set = set(k.lower() for (k,v) in headers)
+ body = escape.utf8(body)
+ if "content-length" not in header_set:
+ headers.append(("Content-Length", str(len(body))))
+ if "content-type" not in header_set:
+ headers.append(("Content-Type", "text/html; charset=UTF-8"))
+ if "server" not in header_set:
+ headers.append(("Server", "TornadoServer/0.1"))
+
+ parts = ["HTTP/1.1 " + data["status"] + "\r\n"]
+ for key, value in headers:
+ parts.append(escape.utf8(key) + ": " + escape.utf8(value) + "\r\n")
+ parts.append("\r\n")
+ parts.append(body)
+ request.write("".join(parts))
+ request.finish()
+ self._log(status_code, request)
+
+ @staticmethod
+ def environ(request):
+ hostport = request.host.split(":")
+ if len(hostport) == 2:
+ host = hostport[0]
+ port = int(hostport[1])
+ else:
+ host = request.host
+ port = 443 if request.protocol == "https" else 80
+ environ = {
+ "REQUEST_METHOD": request.method,
+ "SCRIPT_NAME": "",
+ "PATH_INFO": request.path,
+ "QUERY_STRING": request.query,
+ "REMOTE_ADDR": request.remote_ip,
+ "SERVER_NAME": host,
+ "SERVER_PORT": port,
+ "SERVER_PROTOCOL": request.version,
+ "wsgi.version": (1, 0),
+ "wsgi.url_scheme": request.protocol,
+ "wsgi.input": cStringIO.StringIO(escape.utf8(request.body)),
+ "wsgi.errors": sys.stderr,
+ "wsgi.multithread": False,
+ "wsgi.multiprocess": True,
+ "wsgi.run_once": False,
+ }
+ if "Content-Type" in request.headers:
+ environ["CONTENT_TYPE"] = request.headers["Content-Type"]
+ if "Content-Length" in request.headers:
+ environ["CONTENT_LENGTH"] = request.headers["Content-Length"]
+ for key, value in request.headers.iteritems():
+ environ["HTTP_" + key.replace("-", "_").upper()] = value
+ return environ
+
+ def _log(self, status_code, request):
+ if status_code < 400:
+ log_method = logging.info
+ elif status_code < 500:
+ log_method = logging.warning
+ else:
+ log_method = logging.error
+ request_time = 1000.0 * request.request_time()
+ summary = request.method + " " + request.uri + " (" + \
+ request.remote_ip + ")"
+ log_method("%d %s %.2fms", status_code, summary, request_time)
+
projects / bup.git / commitdiff