From eb7ebb777cc0f285a2df8e357c30cd49af446e13 Mon Sep 17 00:00:00 2001 From: Georg Bauer Date: Sun, 9 Oct 2005 10:47:06 +0000 Subject: [PATCH] i18n: merged r787:r814 from trunk git-svn-id: http://code.djangoproject.com/svn/django/branches/i18n@815 bcc190cf-cafb-0310-a4f2-bffc1f526a37 --- django/bin/django-admin.py | 4 + django/conf/global_settings.py | 3 + django/core/db/backends/mysql.py | 28 ++++- django/core/management.py | 12 +-- django/core/meta/__init__.py | 13 ++- django/core/meta/fields.py | 6 +- django/middleware/cache.py | 116 +++++++++------------ django/middleware/gzip.py | 24 +++++ django/middleware/http.py | 37 +++++++ django/middleware/sessions.py | 2 + django/utils/cache.py | 121 +++++++++++++++++++++ django/utils/decorators.py | 22 ++++ django/views/admin/main.py | 2 +- django/views/decorators/cache.py | 70 +++---------- django/views/decorators/gzip.py | 6 ++ django/views/decorators/http.py | 9 ++ django/views/decorators/vary.py | 35 +++++++ docs/cache.txt | 174 +++++++++++++++++++++++++------ docs/django-admin.txt | 17 ++- docs/middleware.txt | 98 ++++++++++------- tests/runtests.py | 4 +- 21 files changed, 596 insertions(+), 207 deletions(-) create mode 100644 django/middleware/gzip.py create mode 100644 django/middleware/http.py create mode 100644 django/utils/cache.py create mode 100644 django/utils/decorators.py create mode 100644 django/views/decorators/gzip.py create mode 100644 django/views/decorators/http.py create mode 100644 django/views/decorators/vary.py diff --git a/django/bin/django-admin.py b/django/bin/django-admin.py index 31af89dae5..0d021ba172 100755 --- a/django/bin/django-admin.py +++ b/django/bin/django-admin.py @@ -53,11 +53,15 @@ def main(): parser = DjangoOptionParser(get_usage()) parser.add_option('--settings', help='Python path to settings module, e.g. "myproject.settings.main". If this isn\'t provided, the DJANGO_SETTINGS_MODULE environment variable will be used.') + parser.add_option('--pythonpath', + help='Lets you manually add a directory the Python path, e.g. "/home/djangoprojects/myproject".') options, args = parser.parse_args() # Take care of options. if options.settings: os.environ['DJANGO_SETTINGS_MODULE'] = options.settings + if options.pythonpath: + sys.path.insert(0, options.pythonpath) # Run the appropriate action. Unfortunately, optparse can't handle # positional arguments, so this has to parse/validate them. diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py index 6278ae6656..43869e0f1e 100644 --- a/django/conf/global_settings.py +++ b/django/conf/global_settings.py @@ -136,6 +136,8 @@ JING_PATH = "/usr/bin/jing" # response phase the middleware will be applied in reverse order. MIDDLEWARE_CLASSES = ( "django.middleware.sessions.SessionMiddleware", +# "django.middleware.http.ConditionalGetMiddleware", +# "django.middleware.gzip.GZipMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.doc.XViewMiddleware", ) @@ -155,6 +157,7 @@ SESSION_COOKIE_DOMAIN = None # A string like ".lawrence.com", or No # The cache backend to use. See the docstring in django.core.cache for the # possible values. CACHE_BACKEND = 'simple://' +CACHE_MIDDLEWARE_KEY_PREFIX = '' #################### # COMMENTS # diff --git a/django/core/db/backends/mysql.py b/django/core/db/backends/mysql.py index e678740b33..2e77adbc43 100644 --- a/django/core/db/backends/mysql.py +++ b/django/core/db/backends/mysql.py @@ -21,6 +21,32 @@ django_conversions.update({ FIELD_TYPE.TIME: typecasts.typecast_time, }) +# This is an extra debug layer over MySQL queries, to display warnings. +# It's only used when DEBUG=True. +class MysqlDebugWrapper: + def __init__(self, cursor): + self.cursor = cursor + + def execute(self, sql, params=()): + try: + return self.cursor.execute(sql, params) + except Database.Warning, w: + self.cursor.execute("SHOW WARNINGS") + raise Database.Warning, "%s: %s" % (w, self.cursor.fetchall()) + + def executemany(self, sql, param_list): + try: + return self.cursor.executemany(sql, param_list) + except Database.Warning: + self.cursor.execute("SHOW WARNINGS") + raise Database.Warning, "%s: %s" % (w, self.cursor.fetchall()) + + def __getattr__(self, attr): + if self.__dict__.has_key(attr): + return self.__dict__[attr] + else: + return getattr(self.cursor, attr) + class DatabaseWrapper: def __init__(self): self.connection = None @@ -32,7 +58,7 @@ class DatabaseWrapper: self.connection = Database.connect(user=DATABASE_USER, db=DATABASE_NAME, passwd=DATABASE_PASSWORD, host=DATABASE_HOST, conv=django_conversions) if DEBUG: - return base.CursorDebugWrapper(self.connection.cursor(), self) + return base.CursorDebugWrapper(MysqlDebugWrapper(self.connection.cursor()), self) return self.connection.cursor() def commit(self): diff --git a/django/core/management.py b/django/core/management.py index fe7fca6b17..afb498ae63 100644 --- a/django/core/management.py +++ b/django/core/management.py @@ -16,8 +16,8 @@ APP_ARGS = '[modelmodule ...]' # Use django.__path__[0] because we don't know which directory django into # which has been installed. -PROJECT_TEMPLATE_DIR = os.path.join(django.__path__[0], 'conf/%s_template') -ADMIN_TEMPLATE_DIR = os.path.join(django.__path__[0], 'conf/admin_templates') +PROJECT_TEMPLATE_DIR = os.path.join(django.__path__[0], 'conf', '%s_template') +ADMIN_TEMPLATE_DIR = os.path.join(django.__path__[0], 'conf', 'admin_templates') def _get_packages_insert(app_label): return "INSERT INTO packages (label, name) VALUES ('%s', '%s');" % (app_label, app_label) @@ -160,7 +160,7 @@ def get_sql_initial_data(mod): output = [] app_label = mod._MODELS[0]._meta.app_label output.append(_get_packages_insert(app_label)) - app_dir = os.path.normpath(os.path.join(os.path.dirname(mod.__file__), '../sql')) + app_dir = os.path.normpath(os.path.join(os.path.dirname(mod.__file__), '..', 'sql')) for klass in mod._MODELS: opts = klass._meta @@ -376,14 +376,14 @@ def startproject(project_name, directory): _start_helper('project', project_name, directory) # Populate TEMPLATE_DIRS for the admin templates, based on where Django is # installed. - admin_settings_file = os.path.join(directory, project_name, 'settings/admin.py') + admin_settings_file = os.path.join(directory, project_name, 'settings', 'admin.py') settings_contents = open(admin_settings_file, 'r').read() fp = open(admin_settings_file, 'w') settings_contents = re.sub(r'(?s)\b(TEMPLATE_DIRS\s*=\s*\()(.*?)\)', "\\1\n r%r,\\2)" % ADMIN_TEMPLATE_DIR, settings_contents) fp.write(settings_contents) fp.close() # Create a random SECRET_KEY hash, and put it in the main settings. - main_settings_file = os.path.join(directory, project_name, 'settings/main.py') + main_settings_file = os.path.join(directory, project_name, 'settings', 'main.py') settings_contents = open(main_settings_file, 'r').read() fp = open(main_settings_file, 'w') secret_key = ''.join([choice('abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)') for i in range(50)]) @@ -397,7 +397,7 @@ def startapp(app_name, directory): "Creates a Django app for the given app_name in the given directory." # Determine the project_name a bit naively -- by looking at the name of # the parent directory. - project_dir = os.path.normpath(os.path.join(directory, '../')) + project_dir = os.path.normpath(os.path.join(directory, '..')) project_name = os.path.basename(project_dir) _start_helper('app', app_name, directory, project_name) startapp.help_doc = "Creates a Django app directory structure for the given app name in the current directory." diff --git a/django/core/meta/__init__.py b/django/core/meta/__init__.py index 3ca42f14e6..0c6078705a 100644 --- a/django/core/meta/__init__.py +++ b/django/core/meta/__init__.py @@ -1332,16 +1332,19 @@ def function_get_sql_clause(opts, **kwargs): if f == '?': # Special case. order_by.append(db.get_random_function_sql()) else: + if f.startswith('-'): + col_name = f[1:] + order = "DESC" + else: + col_name = f + order = "ASC" # Use the database table as a column prefix if it wasn't given, # and if the requested column isn't a custom SELECT. - if "." not in f and f not in [k[0] for k in kwargs.get('select', [])]: + if "." not in col_name and col_name not in [k[0] for k in kwargs.get('select', [])]: table_prefix = opts.db_table + '.' else: table_prefix = '' - if f.startswith('-'): - order_by.append('%s%s DESC' % (table_prefix, orderfield2column(f[1:], opts))) - else: - order_by.append('%s%s ASC' % (table_prefix, orderfield2column(f, opts))) + order_by.append('%s%s %s' % (table_prefix, orderfield2column(col_name, opts), order)) order_by = ", ".join(order_by) # LIMIT and OFFSET clauses diff --git a/django/core/meta/fields.py b/django/core/meta/fields.py index 3e5cca169d..376595230c 100644 --- a/django/core/meta/fields.py +++ b/django/core/meta/fields.py @@ -596,7 +596,11 @@ class ForeignKey(Field): Field.__init__(self, **kwargs) def get_manipulator_field_objs(self): - return [formfields.IntegerField] + rel_field = self.rel.get_related_field() + if self.rel.raw_id_admin and not isinstance(rel_field, AutoField): + return rel_field.get_manipulator_field_objs() + else: + return [formfields.IntegerField] class ManyToManyField(Field): def __init__(self, to, **kwargs): diff --git a/django/middleware/cache.py b/django/middleware/cache.py index 7f4057eec7..8216c40ae1 100644 --- a/django/middleware/cache.py +++ b/django/middleware/cache.py @@ -1,88 +1,70 @@ +import copy from django.conf import settings from django.core.cache import cache +from django.utils.cache import get_cache_key, learn_cache_key, patch_response_headers from django.utils.httpwrappers import HttpResponseNotModified -from django.utils.text import compress_string -import datetime, md5 class CacheMiddleware: """ Cache middleware. If this is enabled, each Django-powered page will be - cached for CACHE_MIDDLEWARE_SECONDS seconds. Cache is based on URLs. Pages - with GET or POST parameters are not cached. + cached for CACHE_MIDDLEWARE_SECONDS seconds. Cache is based on URLs. - If the cache is shared across multiple sites using the same Django - installation, set the CACHE_MIDDLEWARE_KEY_PREFIX to the name of the site, - or some other string that is unique to this Django instance, to prevent key - collisions. + Only parameter-less GET or HEAD-requests with status code 200 are cached. - This middleware will also make the following optimizations: + This middleware expects that a HEAD request is answered with a response + exactly like the corresponding GET request. - * If the CACHE_MIDDLEWARE_GZIP setting is True, the content will be - gzipped. + When a hit occurs, a shallow copy of the original response object is + returned from process_request. - * ETags will be added, using a simple MD5 hash of the page's content. + Pages will be cached based on the contents of the request headers + listed in the response's "Vary" header. This means that pages shouldn't + change their "Vary" header. + + This middleware also sets ETag, Last-Modified, Expires and Cache-Control + headers on the response object. """ + def __init__(self, cache_timeout=None, key_prefix=None): + self.cache_timeout = cache_timeout + if cache_timeout is None: + self.cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS + self.key_prefix = key_prefix + if key_prefix is None: + self.key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX + def process_request(self, request): - """ - Checks whether the page is already cached. If it is, returns the cached - version. Also handles ETag stuff. - """ - if request.GET or request.POST: - request._cache_middleware_set_cache = False + "Checks whether the page is already cached and returns the cached version if available." + if not request.META['REQUEST_METHOD'] in ('GET', 'HEAD') or request.GET: + request._cache_update_cache = False return None # Don't bother checking the cache. - accept_encoding = '' - if settings.CACHE_MIDDLEWARE_GZIP: - try: - accept_encoding = request.META['HTTP_ACCEPT_ENCODING'] - except KeyError: - pass - accepts_gzip = 'gzip' in accept_encoding - request._cache_middleware_accepts_gzip = accepts_gzip - - # This uses the same cache_key as views.decorators.cache.cache_page, - # so the cache can be shared. - cache_key = 'views.decorators.cache.cache_page.%s.%s.%s' % \ - (settings.CACHE_MIDDLEWARE_KEY_PREFIX, request.path, accepts_gzip) - request._cache_middleware_key = cache_key + cache_key = get_cache_key(request, self.key_prefix) + if cache_key is None: + request._cache_update_cache = True + return None # No cache information available, need to rebuild. response = cache.get(cache_key, None) if response is None: - request._cache_middleware_set_cache = True - return None - else: - request._cache_middleware_set_cache = False - # Logic is from http://simon.incutio.com/archive/2003/04/23/conditionalGet - try: - if_none_match = request.META['HTTP_IF_NONE_MATCH'] - except KeyError: - if_none_match = None - try: - if_modified_since = request.META['HTTP_IF_MODIFIED_SINCE'] - except KeyError: - if_modified_since = None - if if_none_match is None and if_modified_since is None: - pass - elif if_none_match is not None and response['ETag'] != if_none_match: - pass - elif if_modified_since is not None and response['Last-Modified'] != if_modified_since: - pass - else: - return HttpResponseNotModified() - return response + request._cache_update_cache = True + return None # No cache information available, need to rebuild. + + request._cache_update_cache = False + return copy.copy(response) def process_response(self, request, response): - """ - Sets the cache, if needed. - """ - if request._cache_middleware_set_cache: - content = response.get_content_as_string(settings.DEFAULT_CHARSET) - if request._cache_middleware_accepts_gzip: - content = compress_string(content) - response.content = content - response['Content-Encoding'] = 'gzip' - response['ETag'] = md5.new(content).hexdigest() - response['Content-Length'] = '%d' % len(content) - response['Last-Modified'] = datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT') - cache.set(request._cache_middleware_key, response, settings.CACHE_MIDDLEWARE_SECONDS) + "Sets the cache, if needed." + if not request._cache_update_cache: + # We don't need to update the cache, just return. + return response + if not request.META['REQUEST_METHOD'] == 'GET': + # This is a stronger requirement than above. It is needed + # because of interactions between this middleware and the + # HTTPMiddleware, which throws the body of a HEAD-request + # away before this middleware gets a chance to cache it. + return response + if not response.status_code == 200: + return response + patch_response_headers(response, self.cache_timeout) + cache_key = learn_cache_key(request, response, self.cache_timeout, self.key_prefix) + cache.set(cache_key, response, self.cache_timeout) return response diff --git a/django/middleware/gzip.py b/django/middleware/gzip.py new file mode 100644 index 0000000000..201bec2000 --- /dev/null +++ b/django/middleware/gzip.py @@ -0,0 +1,24 @@ +import re +from django.utils.text import compress_string +from django.utils.cache import patch_vary_headers + +re_accepts_gzip = re.compile(r'\bgzip\b') + +class GZipMiddleware: + """ + This middleware compresses content if the browser allows gzip compression. + It sets the Vary header accordingly, so that caches will base their storage + on the Accept-Encoding header. + """ + def process_response(self, request, response): + patch_vary_headers(response, ('Accept-Encoding',)) + if response.has_header('Content-Encoding'): + return response + + ae = request.META.get('HTTP_ACCEPT_ENCODING', '') + if not re_accepts_gzip.search(ae): + return response + + response.content = compress_string(response.content) + response['Content-Encoding'] = 'gzip' + return response diff --git a/django/middleware/http.py b/django/middleware/http.py new file mode 100644 index 0000000000..2bccd60903 --- /dev/null +++ b/django/middleware/http.py @@ -0,0 +1,37 @@ +import datetime + +class ConditionalGetMiddleware: + """ + Handles conditional GET operations. If the response has a ETag or + Last-Modified header, and the request has If-None-Match or + If-Modified-Since, the response is replaced by an HttpNotModified. + + Removes the content from any response to a HEAD request. + + Also sets the Date and Content-Length response-headers. + """ + def process_response(self, request, response): + now = datetime.datetime.utcnow() + response['Date'] = now.strftime('%a, %d %b %Y %H:%M:%S GMT') + if not response.has_header('Content-Length'): + response['Content-Length'] = str(len(response.content)) + + if response.has_header('ETag'): + if_none_match = request.META.get('HTTP_IF_NONE_MATCH', None) + if if_none_match == response['ETag']: + response.status_code = 304 + response.content = '' + response['Content-Length'] = '0' + + if response.has_header('Last-Modified'): + last_mod = response['Last-Modified'] + if_modified_since = request.META.get('HTTP_IF_MODIFIED_SINCE', None) + if if_modified_since == response['Last-Modified']: + response.status_code = 304 + response.content = '' + response['Content-Length'] = '0' + + if request.META['REQUEST_METHOD'] == 'HEAD': + response.content = '' + + return response diff --git a/django/middleware/sessions.py b/django/middleware/sessions.py index a588e3e95b..42b2118410 100644 --- a/django/middleware/sessions.py +++ b/django/middleware/sessions.py @@ -1,5 +1,6 @@ from django.conf.settings import SESSION_COOKIE_NAME, SESSION_COOKIE_AGE, SESSION_COOKIE_DOMAIN from django.models.core import sessions +from django.utils.cache import patch_vary_headers import datetime TEST_COOKIE_NAME = 'testcookie' @@ -61,6 +62,7 @@ class SessionMiddleware: def process_response(self, request, response): # If request.session was modified, or if response.session was set, save # those changes and set a session cookie. + patch_vary_headers(response, ('Cookie',)) try: modified = request.session.modified except AttributeError: diff --git a/django/utils/cache.py b/django/utils/cache.py new file mode 100644 index 0000000000..fcd0825a22 --- /dev/null +++ b/django/utils/cache.py @@ -0,0 +1,121 @@ +""" +This module contains helper functions for controlling caching. It does so by +managing the "Vary" header of responses. It includes functions to patch the +header of response objects directly and decorators that change functions to do +that header-patching themselves. + +For information on the Vary header, see: + + http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.44 + +Essentially, the "Vary" HTTP header defines which headers a cache should take +into account when building its cache key. Requests with the same path but +different header content for headers named in "Vary" need to get different +cache keys to prevent delivery of wrong content. + +A example: i18n middleware would need to distinguish caches by the +"Accept-language" header. +""" + +import datetime, md5, re +from django.conf import settings +from django.core.cache import cache + +vary_delim_re = re.compile(r',\s*') + +def patch_response_headers(response, cache_timeout=None): + """ + Adds some useful headers to the given HttpResponse object: + ETag, Last-Modified, Expires and Cache-Control + + Each header is only added if it isn't already set. + + cache_timeout is in seconds. The CACHE_MIDDLEWARE_SECONDS setting is used + by default. + """ + if cache_timeout is None: + cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS + now = datetime.datetime.utcnow() + expires = now + datetime.timedelta(0, cache_timeout) + if not response.has_header('ETag'): + response['ETag'] = md5.new(response.content).hexdigest() + if not response.has_header('Last-Modified'): + response['Last-Modified'] = now.strftime('%a, %d %b %Y %H:%M:%S GMT') + if not response.has_header('Expires'): + response['Expires'] = expires.strftime('%a, %d %b %Y %H:%M:%S GMT') + if not response.has_header('Cache-Control'): + response['Cache-Control'] = 'max-age=%d' % cache_timeout + +def patch_vary_headers(response, newheaders): + """ + Adds (or updates) the "Vary" header in the given HttpResponse object. + newheaders is a list of header names that should be in "Vary". Existing + headers in "Vary" aren't removed. + """ + # Note that we need to keep the original order intact, because cache + # implementations may rely on the order of the Vary contents in, say, + # computing an MD5 hash. + vary = [] + if response.has_header('Vary'): + vary = vary_delim_re.split(response['Vary']) + oldheaders = dict([(el.lower(), 1) for el in vary]) + for newheader in newheaders: + if not newheader.lower() in oldheaders: + vary.append(newheader) + response['Vary'] = ', '.join(vary) + +def _generate_cache_key(request, headerlist, key_prefix): + "Returns a cache key from the headers given in the header list." + ctx = md5.new() + for header in headerlist: + value = request.META.get(header, None) + if value is not None: + ctx.update(value) + return 'views.decorators.cache.cache_page.%s.%s.%s' % (key_prefix, request.path, ctx.hexdigest()) + +def get_cache_key(request, key_prefix=None): + """ + Returns a cache key based on the request path. It can be used in the + request phase because it pulls the list of headers to take into account + from the global path registry and uses those to build a cache key to check + against. + + If there is no headerlist stored, the page needs to be rebuilt, so this + function returns None. + """ + if key_prefix is None: + key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX + cache_key = 'views.decorators.cache.cache_header.%s.%s' % (key_prefix, request.path) + headerlist = cache.get(cache_key, None) + if headerlist is not None: + return _generate_cache_key(request, headerlist, key_prefix) + else: + return None + +def learn_cache_key(request, response, cache_timeout=None, key_prefix=None): + """ + Learns what headers to take into account for some request path from the + response object. It stores those headers in a global path registry so that + later access to that path will know what headers to take into account + without building the response object itself. The headers are named in the + Vary header of the response, but we want to prevent response generation. + + The list of headers to use for cache key generation is stored in the same + cache as the pages themselves. If the cache ages some data out of the + cache, this just means that we have to build the response once to get at + the Vary header and so at the list of headers to use for the cache key. + """ + if key_prefix is None: + key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX + if cache_timeout is None: + cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS + cache_key = 'views.decorators.cache.cache_header.%s.%s' % (key_prefix, request.path) + if response.has_header('Vary'): + headerlist = ['HTTP_'+header.upper().replace('-', '_') for header in vary_delim_re.split(response['Vary'])] + cache.set(cache_key, headerlist, cache_timeout) + return _generate_cache_key(request, headerlist, key_prefix) + else: + # if there is no Vary header, we still need a cache key + # for the request.path + cache.set(cache_key, [], cache_timeout) + return _generate_cache_key(request, [], key_prefix) diff --git a/django/utils/decorators.py b/django/utils/decorators.py new file mode 100644 index 0000000000..b21a4e4248 --- /dev/null +++ b/django/utils/decorators.py @@ -0,0 +1,22 @@ +"Functions that help with dynamically creating decorators for views." + +def decorator_from_middleware(middleware_class): + """ + Given a middleware class (not an instance), returns a view decorator. This + lets you use middleware functionality on a per-view basis. + """ + def _decorator_from_middleware(view_func, *args, **kwargs): + middleware = middleware_class(*args, **kwargs) + def _wrapped_view(request, *args, **kwargs): + if hasattr(middleware, 'process_request'): + result = middleware.process_request(request) + if result is not None: + return result + response = view_func(request, *args, **kwargs) + if hasattr(middleware, 'process_response'): + result = middleware.process_response(request, response) + if result is not None: + return result + return response + return _wrapped_view + return _decorator_from_middleware diff --git a/django/views/admin/main.py b/django/views/admin/main.py index c58da1fbca..5379c8efb5 100644 --- a/django/views/admin/main.py +++ b/django/views/admin/main.py @@ -251,7 +251,7 @@ def change_list(request, app_label, module_name): lookup_val = request.GET.get(lookup_kwarg, None) lookup_val2 = request.GET.get(lookup_kwarg2, None) filter_template.append('

By %s: