Merged Unicode branch into trunk (r4952:5608). This should be fully

backwards compatible for all practical purposes.

Fixed #2391, #2489, #2996, #3322, #3344, #3370, #3406, #3432, #3454, #3492, #3582, #3690, #3878, #3891, #3937, #4039, #4141, #4227, #4286, #4291, #4300, #4452, #4702


git-svn-id: http://code.djangoproject.com/svn/django/trunk@5609 bcc190cf-cafb-0310-a4f2-bffc1f526a37

AUTHORS

@@ -113,6 +113,7 @@ answer newbie questions, and generally made Django that much better:
     Simon Greenhill <dev@simon.net.nz>
     Owen Griffiths
     Espen Grindhaug <http://grindhaug.org/>
+    Thomas Güttler <hv@tbz-pariv.de>
     Brian Harring <ferringb@gmail.com>
     Brant Harris
     Hawkeye
@@ -147,6 +148,7 @@ answer newbie questions, and generally made Django that much better:
     Bruce Kroeze <http://coderseye.com/>
     Joseph Kocherhans
     konrad@gwu.edu
+    kurtiss@meetro.com
     lakin.wecker@gmail.com
     Nick Lane <nick.lane.au@gmail.com>
     Stuart Langridge <http://www.kryogenix.org/>

django/bin/make-messages.py

@@ -103,7 +103,7 @@ def make_messages():
                         open(os.path.join(dirpath, '%s.py' % file), "wb").write(templatize(src))
                         thefile = '%s.py' % file
                     if verbose: sys.stdout.write('processing file %s in %s\n' % (file, dirpath))
-                    cmd = 'xgettext %s -d %s -L Python --keyword=gettext_noop --keyword=gettext_lazy --keyword=ngettext_lazy:1,2 --from-code UTF-8 -o - "%s"' % (
+                    cmd = 'xgettext %s -d %s -L Python --keyword=gettext_noop --keyword=gettext_lazy --keyword=ngettext_lazy:1,2 --keyword=ugettext_noop --keyword=ugettext_lazy --keyword=ungettext_lazy:1,2 --from-code UTF-8 -o - "%s"' % (
                         os.path.exists(potfile) and '--omit-header' or '', domain, os.path.join(dirpath, thefile))
                     (stdin, stdout, stderr) = os.popen3(cmd, 'b')
                     msgs = stdout.read()

django/conf/global_settings.py

@@ -97,6 +97,9 @@ MANAGERS = ADMINS
 DEFAULT_CONTENT_TYPE = 'text/html'
 DEFAULT_CHARSET = 'utf-8'
 
+# Encoding of files read from disk (template and initial SQL files).
+FILE_CHARSET = 'utf-8'
+
 # E-mail address that error messages come from.
 SERVER_EMAIL = 'root@localhost'
 

django/contrib/admin/filterspecs.py

@@ -7,6 +7,8 @@ certain test -- e.g. being a DateField or ForeignKey.
 """
 
 from django.db import models
+from django.utils.encoding import smart_unicode, iri_to_uri
+from django.utils.translation import ugettext as _
 import datetime
 
 class FilterSpec(object):
@@ -37,12 +39,12 @@ class FilterSpec(object):
     def output(self, cl):
         t = []
         if self.has_output():
-            t.append(_('<h3>By %s:</h3>\n<ul>\n') % self.title())
+            t.append(_(u'<h3>By %s:</h3>\n<ul>\n') % self.title())
 
             for choice in self.choices(cl):
-                t.append('<li%s><a href="%s">%s</a></li>\n' % \
+                t.append(u'<li%s><a href="%s">%s</a></li>\n' % \
                     ((choice['selected'] and ' class="selected"' or ''),
-                     choice['query_string'] ,
+                     iri_to_uri(choice['query_string']),
                      choice['display']))
             t.append('</ul>\n\n')
         return "".join(t)
@@ -70,7 +72,7 @@ class RelatedFilterSpec(FilterSpec):
                'display': _('All')}
         for val in self.lookup_choices:
             pk_val = getattr(val, self.field.rel.to._meta.pk.attname)
-            yield {'selected': self.lookup_val == str(pk_val),
+            yield {'selected': self.lookup_val == smart_unicode(pk_val),
                    'query_string': cl.get_query_string({self.lookup_kwarg: pk_val}),
                    'display': val}
 
@@ -87,7 +89,7 @@ class ChoicesFilterSpec(FilterSpec):
                'query_string': cl.get_query_string({}, [self.lookup_kwarg]),
                'display': _('All')}
         for k, v in self.field.choices:
-            yield {'selected': str(k) == self.lookup_val,
+            yield {'selected': smart_unicode(k) == self.lookup_val,
                     'query_string': cl.get_query_string({self.lookup_kwarg: k}),
                     'display': v}
 
@@ -168,7 +170,7 @@ class AllValuesFilterSpec(FilterSpec):
                'query_string': cl.get_query_string({}, [self.field.name]),
                'display': _('All')}
         for val in self.lookup_choices:
-            val = str(val[self.field.name])
+            val = smart_unicode(val[self.field.name])
             yield {'selected': self.lookup_val == val,
                    'query_string': cl.get_query_string({self.field.name: val}),
                    'display': val}

django/contrib/admin/media/js/urlify.js

@@ -1,15 +1,110 @@
+var LATIN_MAP =
+{
+    'À': 'A', 'Á': 'A', 'Â': 'A', 'Ã': 'A', 'Ä': 'A', 'Å': 'A', 'Æ': 'AE', 'Ç':
+    'C', 'È': 'E', 'É': 'E', 'Ê': 'E', 'Ë': 'E', 'Ì': 'I', 'Í': 'I', 'Î': 'I',
+    'Ï': 'I', 'Ð': 'D', 'Ñ': 'N', 'Ò': 'O', 'Ó': 'O', 'Ô': 'O', 'Õ': 'O', 'Ö':
+    'O', 'Ø': 'O', 'Ù': 'U', 'Ú': 'U', 'Û': 'U', 'Ü': 'U', 'Ý': 'Y', 'Þ': 'TH',
+    'ß': 'ss', 'à':'a', 'á':'a', 'â': 'a', 'ã': 'a', 'ä': 'a', 'å': 'a', 'æ':
+    'ae', 'ç': 'c', 'è': 'e', 'é': 'e', 'ê': 'e', 'ë': 'e', 'ì': 'i', 'í': 'i',
+    'î': 'i', 'ï': 'i', 'ð': 'o', 'ñ': 'n', 'ò': 'o', 'ó': 'o', 'ô': 'o', 'õ':
+    'o', 'ö': 'o', 'ø': 'o', 'ù': 'u', 'ú': 'u', 'û': 'u', 'ü': 'u', 'ý': 'y',
+    'þ': 'th', 'ÿ': 'y',
+}
+var LATIN_SYMBOLS_MAP =
+{
+    '©':'(c)',
+}
+var GREEK_MAP =
+{
+    'α':'a', 'β':'b', 'γ':'g', 'δ':'d', 'ε':'e', 'ζ':'z', 'η':'h', 'θ':'8',
+    'ι':'i', 'κ':'k', 'λ':'l', 'μ':'m', 'ν':'n', 'ξ':'3', 'ο':'o', 'π':'p',
+    'ρ':'r', 'σ':'s', 'τ':'t', 'υ':'y', 'φ':'f', 'χ':'x', 'ψ':'ps', 'ω':'w',
+    'ά':'a', 'έ':'e', 'ί':'i', 'ό':'o', 'ύ':'y', 'ή':'h', 'ώ':'w', 'ς':'s',
+    'ϊ':'i', 'ΰ':'y', 'ϋ':'y', 'ΐ':'i',
+    'Α':'A', 'Β':'B', 'Γ':'G', 'Δ':'D', 'Ε':'E', 'Ζ':'Z', 'Η':'H', 'Θ':'8',
+    'Ι':'I', 'Κ':'K', 'Λ':'L', 'Μ':'M', 'Ν':'N', 'Ξ':'3', 'Ο':'O', 'Π':'P',
+    'Ρ':'R', 'Σ':'S', 'Τ':'T', 'Υ':'Y', 'Φ':'F', 'Χ':'X', 'Ψ':'PS', 'Ω':'W',
+    'Ά':'A', 'Έ':'E', 'Ί':'I', 'Ό':'O', 'Ύ':'Y', 'Ή':'H', 'Ώ':'W', 'Ϊ':'I',
+    'Ϋ':'Y'
+}
+var TURKISH_MAP = {
+    'ş':'s', 'Ş':'S', 'ı':'i', 'İ':'I', 'ç':'c', 'Ç':'C', 'ü':'u', 'Ü':'U',
+    'ö':'o', 'Ö':'O', 'ğ':'g', 'Ğ':'G',
+}
+// var RUSSIAN_MAP =
+// {
+// }
+
+var ALL_DOWNCODE_MAPS=new Array()
+ALL_DOWNCODE_MAPS[0]=LATIN_MAP
+ALL_DOWNCODE_MAPS[1]=LATIN_SYMBOLS_MAP
+ALL_DOWNCODE_MAPS[2]=GREEK_MAP
+ALL_DOWNCODE_MAPS[3]=TURKISH_MAP
+//ALL_DOWNCODE_MAPS[4]=RUSSIAN_MAP
+
+var Downcoder = new Object();
+Downcoder.Initialize = function()
+{
+    if (Downcoder.map) // already made
+        return ;
+    Downcoder.map ={}
+    Downcoder.chars = '' ;
+    for(var i in ALL_DOWNCODE_MAPS)
+    {
+        var lookup = ALL_DOWNCODE_MAPS[i]
+        for (var c in lookup)
+        {
+            Downcoder.map[c] = lookup[c] ;
+            Downcoder.chars += c ;
+        }
+     }
+    Downcoder.regex = new RegExp('[' + Downcoder.chars + ']|[^' + Downcoder.chars + ']+','g') ;
+}
+
+downcode= function( slug )
+{
+    Downcoder.Initialize() ;
+    var downcoded =""
+    var pieces = slug.match(Downcoder.regex);
+    if(pieces)
+    {
+        for (var i = 0 ; i < pieces.length ; i++)
+        {
+            if (pieces[i].length == 1)
+            {
+                var mapped = Downcoder.map[pieces[i]] ;
+                if (mapped != null)
+                {
+                    downcoded+=mapped;
+                    continue ;
+                }
+            }
+            downcoded+=pieces[i];
+        }
+    }
+    else
+    {
+        downcoded = slug;
+    }
+    return downcoded;
+}
+
+
 function URLify(s, num_chars) {
     // changes, e.g., "Petty theft" to "petty_theft"
     // remove all these words from the string before urlifying
+    s = downcode(s);
     removelist = ["a", "an", "as", "at", "before", "but", "by", "for", "from",
                   "is", "in", "into", "like", "of", "off", "on", "onto", "per",
                   "since", "than", "the", "this", "that", "to", "up", "via",
                   "with"];
     r = new RegExp('\\b(' + removelist.join('|') + ')\\b', 'gi');
     s = s.replace(r, '');
+    // if downcode doesn't hit, the char will be stripped here
     s = s.replace(/[^-\w\s]/g, '');  // remove unneeded chars
     s = s.replace(/^\s+|\s+$/g, ''); // trim leading/trailing spaces
     s = s.replace(/[-\s]+/g, '-');   // convert spaces to hyphens
     s = s.toLowerCase();             // convert to lowercase
     return s.substring(0, num_chars);// trim to first num_chars chars
 }
+

django/contrib/admin/models.py

@@ -1,7 +1,8 @@
 from django.db import models
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.auth.models import User
-from django.utils.translation import gettext_lazy as _
+from django.utils.translation import ugettext_lazy as _
+from django.utils.encoding import smart_unicode
 
 ADDITION = 1
 CHANGE = 2
@@ -9,7 +10,7 @@ DELETION = 3
 
 class LogEntryManager(models.Manager):
     def log_action(self, user_id, content_type_id, object_id, object_repr, action_flag, change_message=''):
-        e = self.model(None, None, user_id, content_type_id, str(object_id), object_repr[:200], action_flag, change_message)
+        e = self.model(None, None, user_id, content_type_id, smart_unicode(object_id), object_repr[:200], action_flag, change_message)
         e.save()
 
 class LogEntry(models.Model):
@@ -28,7 +29,7 @@ class LogEntry(models.Model):
         ordering = ('-action_time',)
 
     def __repr__(self):
-        return str(self.action_time)
+        return smart_unicode(self.action_time)
 
     def is_addition(self):
         return self.action_flag == ADDITION
@@ -48,4 +49,4 @@ class LogEntry(models.Model):
         Returns the admin URL to edit the object represented by this log entry.
         This is relative to the Django admin index page.
         """
-        return "%s/%s/%s/" % (self.content_type.app_label, self.content_type.model, self.object_id)
+        return u"%s/%s/%s/" % (self.content_type.app_label, self.content_type.model, self.object_id)

django/contrib/admin/templates/admin/filter.html

@@ -3,6 +3,6 @@
 <ul>
 {% for choice in choices %}
     <li{% if choice.selected %} class="selected"{% endif %}>
-    <a href="{{ choice.query_string }}">{{ choice.display|escape }}</a></li>
+    <a href="{{ choice.query_string|iriencode }}">{{ choice.display|escape }}</a></li>
 {% endfor %}
 </ul>

django/contrib/admin/templatetags/admin_list.py

@@ -6,7 +6,8 @@ from django.db import models
 from django.utils import dateformat
 from django.utils.html import escape
 from django.utils.text import capfirst
-from django.utils.translation import get_date_formats, get_partial_date_formats
+from django.utils.translation import get_date_formats, get_partial_date_formats, ugettext as _
+from django.utils.encoding import smart_unicode, smart_str, force_unicode
 from django.template import Library
 import datetime
 
@@ -16,11 +17,11 @@ DOT = '.'
 
 def paginator_number(cl,i):
     if i == DOT:
-        return '... '
+        return u'... '
     elif i == cl.page_num:
-        return '<span class="this-page">%d</span> ' % (i+1)
+        return u'<span class="this-page">%d</span> ' % (i+1)
     else:
-        return '<a href="%s"%s>%d</a> ' % (cl.get_query_string({PAGE_VAR: i}), (i == cl.paginator.pages-1 and ' class="end"' or ''), i+1)
+        return u'<a href="%s"%s>%d</a> ' % (cl.get_query_string({PAGE_VAR: i}), (i == cl.paginator.pages-1 and ' class="end"' or ''), i+1)
 paginator_number = register.simple_tag(paginator_number)
 
 def pagination(cl):
@@ -75,10 +76,12 @@ def result_headers(cl):
             admin_order_field = None
         except models.FieldDoesNotExist:
             # For non-field list_display values, check for the function
-            # attribute "short_description". If that doesn't exist, fall
+            # attribute "short_description". If that doesn't exist, fall back
-            # back to the method name. And __str__ is a special-case.
+            # to the method name. And __str__ and __unicode__ are special-cases.
-            if field_name == '__str__':
+            if field_name == '__unicode__':
-                header = lookup_opts.verbose_name
+                header = force_unicode(lookup_opts.verbose_name)
+            elif field_name == '__str__':
+                header = smart_str(lookup_opts.verbose_name)
             else:
                 attr = getattr(cl.model, field_name) # Let AttributeErrors propagate.
                 try:
@@ -114,7 +117,7 @@ def result_headers(cl):
 
 def _boolean_icon(field_val):
     BOOLEAN_MAPPING = {True: 'yes', False: 'no', None: 'unknown'}
-    return '<img src="%simg/admin/icon-%s.gif" alt="%s" />' % (settings.ADMIN_MEDIA_PREFIX, BOOLEAN_MAPPING[field_val], field_val)
+    return u'<img src="%simg/admin/icon-%s.gif" alt="%s" />' % (settings.ADMIN_MEDIA_PREFIX, BOOLEAN_MAPPING[field_val], field_val)
 
 def items_for_result(cl, result):
     first = True
@@ -136,7 +139,7 @@ def items_for_result(cl, result):
                     allow_tags = True
                     result_repr = _boolean_icon(attr)
                 else:
-                    result_repr = str(attr)
+                    result_repr = smart_unicode(attr)
             except (AttributeError, ObjectDoesNotExist):
                 result_repr = EMPTY_CHANGELIST_VALUE
             else:
@@ -179,19 +182,19 @@ def items_for_result(cl, result):
             elif f.choices:
                 result_repr = dict(f.choices).get(field_val, EMPTY_CHANGELIST_VALUE)
             else:
-                result_repr = escape(str(field_val))
+                result_repr = escape(field_val)
-        if result_repr == '':
+        if force_unicode(result_repr) == '':
             result_repr = '&nbsp;'
         # If list_display_links not defined, add the link tag to the first field
-        if (first and not cl.lookup_opts.admin.list_display_links) or field_name in cl.lookup_opts.admin.list_display_links: 
+        if (first and not cl.lookup_opts.admin.list_display_links) or field_name in cl.lookup_opts.admin.list_display_links:
             table_tag = {True:'th', False:'td'}[first]
             first = False
             url = cl.url_for_result(result)
-            result_id = str(getattr(result, pk)) # str() is needed in case of 23L (long ints)
+            result_id = smart_unicode(getattr(result, pk)) # conversion to string is needed in case of 23L (long ints)
-            yield ('<%s%s><a href="%s"%s>%s</a></%s>' % \
+            yield (u'<%s%s><a href="%s"%s>%s</a></%s>' % \
                 (table_tag, row_class, url, (cl.is_popup and ' onclick="opener.dismissRelatedLookupPopup(window, %r); return false;"' % result_id or ''), result_repr, table_tag))
         else:
-            yield ('<td%s>%s</td>' % (row_class, result_repr))
+            yield (u'<td%s>%s</td>' % (row_class, result_repr))
 
 def results(cl):
     for res in cl.result_list:

django/contrib/admin/templatetags/admin_modify.py

@@ -2,6 +2,7 @@ from django import template
 from django.contrib.admin.views.main import AdminBoundField
 from django.template import loader
 from django.utils.text import capfirst
+from django.utils.encoding import force_unicode
 from django.db import models
 from django.db.models.fields import Field
 from django.db.models.related import BoundRelatedObject
@@ -14,7 +15,7 @@ word_re = re.compile('[A-Z][a-z]+')
 absolute_url_re = re.compile(r'^(?:http(?:s)?:/)?/', re.IGNORECASE)
 
 def class_name_to_underscored(name):
-    return '_'.join([s.lower() for s in word_re.findall(name)[:-1]])
+    return u'_'.join([s.lower() for s in word_re.findall(name)[:-1]])
 
 def include_admin_script(script_path):
     """
@@ -31,7 +32,7 @@ def include_admin_script(script_path):
     """
     if not absolute_url_re.match(script_path):
         script_path = '%s%s' % (settings.ADMIN_MEDIA_PREFIX, script_path)
-    return '<script type="text/javascript" src="%s"></script>' % script_path
+    return u'<script type="text/javascript" src="%s"></script>' % script_path
 include_admin_script = register.simple_tag(include_admin_script)
 
 def submit_row(context):
@@ -61,9 +62,9 @@ def field_label(bound_field):
         if not bound_field.first:
             class_names.append('inline')
         colon = ":"
-    class_str = class_names and ' class="%s"' % ' '.join(class_names) or ''
+    class_str = class_names and u' class="%s"' % u' '.join(class_names) or u''
-    return '<label for="%s"%s>%s%s</label> ' % (bound_field.element_id, class_str, \
+    return u'<label for="%s"%s>%s%s</label> ' % (bound_field.element_id, class_str, \
-        capfirst(bound_field.field.verbose_name), colon)
+        force_unicode(capfirst(bound_field.field.verbose_name)), colon)
 field_label = register.simple_tag(field_label)
 
 class FieldWidgetNode(template.Node):
@@ -77,7 +78,7 @@ class FieldWidgetNode(template.Node):
         if klass not in cls.nodelists:
             try:
                 field_class_name = klass.__name__
-                template_name = "widget/%s.html" % class_name_to_underscored(field_class_name)
+                template_name = u"widget/%s.html" % class_name_to_underscored(field_class_name)
                 nodelist = loader.get_template(template_name).nodelist
             except template.TemplateDoesNotExist:
                 super_klass = bool(klass.__bases__) and klass.__bases__[0] or None
@@ -175,30 +176,30 @@ class EditInlineNode(template.Node):
         return output
 
 def output_all(form_fields):
-    return ''.join([str(f) for f in form_fields])
+    return u''.join([force_unicode(f) for f in form_fields])
 output_all = register.simple_tag(output_all)
 
 def auto_populated_field_script(auto_pop_fields, change = False):
     t = []
     for field in auto_pop_fields:
         if change:
-            t.append('document.getElementById("id_%s")._changed = true;' % field.name)
+            t.append(u'document.getElementById("id_%s")._changed = true;' % field.name)
         else:
-            t.append('document.getElementById("id_%s").onchange = function() { this._changed = true; };' % field.name)
+            t.append(u'document.getElementById("id_%s").onchange = function() { this._changed = true; };' % field.name)
 
-        add_values = ' + " " + '.join(['document.getElementById("id_%s").value' % g for g in field.prepopulate_from])
+        add_values = u' + " " + '.join([u'document.getElementById("id_%s").value' % g for g in field.prepopulate_from])
         for f in field.prepopulate_from:
-            t.append('document.getElementById("id_%s").onkeyup = function() {' \
+            t.append(u'document.getElementById("id_%s").onkeyup = function() {' \
                      ' var e = document.getElementById("id_%s");' \
                      ' if(!e._changed) { e.value = URLify(%s, %s);} }; ' % (
                      f, field.name, add_values, field.maxlength))
-    return ''.join(t)
+    return u''.join(t)
 auto_populated_field_script = register.simple_tag(auto_populated_field_script)
 
 def filter_interface_script_maybe(bound_field):
     f = bound_field.field
     if f.rel and isinstance(f.rel, models.ManyToManyRel) and f.rel.filter_interface:
-        return '<script type="text/javascript">addEvent(window, "load", function(e) {' \
+        return u'<script type="text/javascript">addEvent(window, "load", function(e) {' \
               ' SelectFilter.init("id_%s", "%s", %s, "%s"); });</script>\n' % (
               f.name, f.verbose_name.replace('"', '\\"'), f.rel.filter_interface-1, settings.ADMIN_MEDIA_PREFIX)
     else:

django/contrib/admin/templatetags/adminapplist.py

@@ -1,5 +1,6 @@
 from django import template
 from django.db.models import get_models
+from django.utils.encoding import force_unicode
 
 register = template.Library()
 
@@ -36,8 +37,8 @@ class AdminApplistNode(template.Node):
                         # If so, add the module to the model_list.
                         if True in perms.values():
                             model_list.append({
-                                'name': capfirst(m._meta.verbose_name_plural),
+                                'name': force_unicode(capfirst(m._meta.verbose_name_plural)),
-                                'admin_url': '%s/%s/' % (app_label, m.__name__.lower()),
+                                'admin_url': u'%s/%s/' % (force_unicode(app_label), m.__name__.lower()),
                                 'perms': perms,
                             })
 

django/contrib/admin/views/auth.py

@@ -6,6 +6,7 @@ from django import oldforms, template
 from django.shortcuts import render_to_response, get_object_or_404
 from django.http import HttpResponseRedirect
 from django.utils.html import escape
+from django.utils.translation import ugettext as _
 
 def user_add_stage(request):
     if not request.user.has_perm('auth.change_user'):

django/contrib/admin/views/decorators.py

@@ -3,11 +3,11 @@ from django.conf import settings
 from django.contrib.auth.models import User
 from django.contrib.auth import authenticate, login
 from django.shortcuts import render_to_response
-from django.utils.translation import gettext_lazy
+from django.utils.translation import ugettext_lazy, ugettext as _
 import base64, datetime, md5
 import cPickle as pickle
 
-ERROR_MESSAGE = gettext_lazy("Please enter a correct username and password. Note that both fields are case-sensitive.")
+ERROR_MESSAGE = ugettext_lazy("Please enter a correct username and password. Note that both fields are case-sensitive.")
 LOGIN_FORM_KEY = 'this_is_the_login_form'
 
 def _display_login_form(request, error_message=''):

django/contrib/admin/views/doc.py

@@ -9,6 +9,7 @@ from django.http import Http404, get_host
 from django.core import urlresolvers
 from django.contrib.admin import utils
 from django.contrib.sites.models import Site
+from django.utils.translation import ugettext as _
 import inspect, os, re
 
 # Exclude methods starting with these strings from documentation

django/contrib/admin/views/main.py

@@ -12,6 +12,8 @@ from django.db.models.query import handle_legacy_orderlist, QuerySet
 from django.http import Http404, HttpResponse, HttpResponseRedirect
 from django.utils.html import escape
 from django.utils.text import capfirst, get_text_list
+from django.utils.encoding import force_unicode, smart_str
+from django.utils.translation import ugettext as _
 import operator
 
 try:
@@ -130,11 +132,11 @@ class AdminBoundField(object):
         if max([bool(f.errors()) for f in self.form_fields]):
             classes.append('error')
         if classes:
-            self.cell_class_attribute = ' class="%s" ' % ' '.join(classes)
+            self.cell_class_attribute = u' class="%s" ' % ' '.join(classes)
         self._repr_filled = False
 
         if field.rel:
-            self.related_url = '../../../%s/%s/' % (field.rel.to._meta.app_label, field.rel.to._meta.object_name.lower())
+            self.related_url = u'../../../%s/%s/' % (field.rel.to._meta.app_label, field.rel.to._meta.object_name.lower())
 
     def original_value(self):
         if self.original:
@@ -145,9 +147,9 @@ class AdminBoundField(object):
             return self._display
         except AttributeError:
             if isinstance(self.field.rel, models.ManyToOneRel):
-                self._display = getattr(self.original, self.field.name)
+                self._display = force_unicode(getattr(self.original, self.field.name), strings_only=True)
             elif isinstance(self.field.rel, models.ManyToManyRel):
-                self._display = ", ".join([str(obj) for obj in getattr(self.original, self.field.name).all()])
+                self._display = u", ".join([force_unicode(obj) for obj in getattr(self.original, self.field.name).all()])
             return self._display
 
     def __repr__(self):
@@ -258,8 +260,8 @@ def add_stage(request, app_label, model_name, show_delete=False, form_url='', po
         if not errors:
             new_object = manipulator.save(new_data)
             pk_value = new_object._get_pk_val()
-            LogEntry.objects.log_action(request.user.id, ContentType.objects.get_for_model(model).id, pk_value, str(new_object), ADDITION)
+            LogEntry.objects.log_action(request.user.id, ContentType.objects.get_for_model(model).id, pk_value, force_unicode(new_object), ADDITION)
-            msg = _('The %(name)s "%(obj)s" was added successfully.') % {'name': opts.verbose_name, 'obj': new_object}
+            msg = _('The %(name)s "%(obj)s" was added successfully.') % {'name': force_unicode(opts.verbose_name), 'obj': new_object}
             # Here, we distinguish between different save types by checking for
             # the presence of keys in request.POST.
             if "_continue" in request.POST:
@@ -271,9 +273,9 @@ def add_stage(request, app_label, model_name, show_delete=False, form_url='', po
                 if type(pk_value) is str: # Quote if string, so JavaScript doesn't think it's a variable.
                     pk_value = '"%s"' % pk_value.replace('"', '\\"')
                 return HttpResponse('<script type="text/javascript">opener.dismissAddAnotherPopup(window, %s, "%s");</script>' % \
-                    (pk_value, str(new_object).replace('"', '\\"')))
+                    (pk_value, force_unicode(new_object).replace('"', '\\"')))
             elif "_addanother" in request.POST:
-                request.user.message_set.create(message=msg + ' ' + (_("You may add another %s below.") % opts.verbose_name))
+                request.user.message_set.create(message=msg + ' ' + (_("You may add another %s below.") % force_unicode(opts.verbose_name)))
                 return HttpResponseRedirect(request.path)
             else:
                 request.user.message_set.create(message=msg)
@@ -291,7 +293,7 @@ def add_stage(request, app_label, model_name, show_delete=False, form_url='', po
     form = oldforms.FormWrapper(manipulator, new_data, errors)
 
     c = template.RequestContext(request, {
-        'title': _('Add %s') % opts.verbose_name,
+        'title': _('Add %s') % force_unicode(opts.verbose_name),
         'form': form,
         'is_popup': '_popup' in request.REQUEST,
         'show_delete': show_delete,
@@ -345,9 +347,9 @@ def change_stage(request, app_label, model_name, object_id):
             change_message = ' '.join(change_message)
             if not change_message:
                 change_message = _('No fields changed.')
-            LogEntry.objects.log_action(request.user.id, ContentType.objects.get_for_model(model).id, pk_value, str(new_object), CHANGE, change_message)
+            LogEntry.objects.log_action(request.user.id, ContentType.objects.get_for_model(model).id, pk_value, force_unicode(new_object), CHANGE, change_message)
 
-            msg = _('The %(name)s "%(obj)s" was changed successfully.') % {'name': opts.verbose_name, 'obj': new_object}
+            msg = _('The %(name)s "%(obj)s" was changed successfully.') % {'name': force_unicode(opts.verbose_name), 'obj': new_object}
             if "_continue" in request.POST:
                 request.user.message_set.create(message=msg + ' ' + _("You may edit it again below."))
                 if '_popup' in request.REQUEST:
@@ -355,10 +357,10 @@ def change_stage(request, app_label, model_name, object_id):
                 else:
                     return HttpResponseRedirect(request.path)
             elif "_saveasnew" in request.POST:
-                request.user.message_set.create(message=_('The %(name)s "%(obj)s" was added successfully. You may edit it again below.') % {'name': opts.verbose_name, 'obj': new_object})
+                request.user.message_set.create(message=_('The %(name)s "%(obj)s" was added successfully. You may edit it again below.') % {'name': force_unicode(opts.verbose_name), 'obj': new_object})
                 return HttpResponseRedirect("../%s/" % pk_value)
             elif "_addanother" in request.POST:
-                request.user.message_set.create(message=msg + ' ' + (_("You may add another %s below.") % opts.verbose_name))
+                request.user.message_set.create(message=msg + ' ' + (_("You may add another %s below.") % force_unicode(opts.verbose_name)))
                 return HttpResponseRedirect("../add/")
             else:
                 request.user.message_set.create(message=msg)
@@ -393,7 +395,7 @@ def change_stage(request, app_label, model_name, object_id):
             form.order_objects.extend(orig_list)
 
     c = template.RequestContext(request, {
-        'title': _('Change %s') % opts.verbose_name,
+        'title': _('Change %s') % force_unicode(opts.verbose_name),
         'form': form,
         'object_id': object_id,
         'original': manipulator.original_object,
@@ -434,11 +436,11 @@ def _get_deleted_objects(deleted_objects, perms_needed, user, obj, opts, current
                 if related.field.rel.edit_inline or not related.opts.admin:
                     # Don't display link to edit, because it either has no
                     # admin or is edited inline.
-                    nh(deleted_objects, current_depth, ['%s: %s' % (capfirst(related.opts.verbose_name), sub_obj), []])
+                    nh(deleted_objects, current_depth, [u'%s: %s' % (force_unicode(capfirst(related.opts.verbose_name)), sub_obj), []])
                 else:
                     # Display a link to the admin page.
-                    nh(deleted_objects, current_depth, ['%s: <a href="../../../../%s/%s/%s/">%s</a>' % \
+                    nh(deleted_objects, current_depth, [u'%s: <a href="../../../../%s/%s/%s/">%s</a>' % \
-                        (capfirst(related.opts.verbose_name), related.opts.app_label, related.opts.object_name.lower(),
+                        (force_unicode(capfirst(related.opts.verbose_name)), related.opts.app_label, related.opts.object_name.lower(),
                         sub_obj._get_pk_val(), sub_obj), []])
                 _get_deleted_objects(deleted_objects, perms_needed, user, sub_obj, related.opts, current_depth+2)
         else:
@@ -448,11 +450,11 @@ def _get_deleted_objects(deleted_objects, perms_needed, user, obj, opts, current
                 if related.field.rel.edit_inline or not related.opts.admin:
                     # Don't display link to edit, because it either has no
                     # admin or is edited inline.
-                    nh(deleted_objects, current_depth, ['%s: %s' % (capfirst(related.opts.verbose_name), escape(str(sub_obj))), []])
+                    nh(deleted_objects, current_depth, [u'%s: %s' % (force_unicode(capfirst(related.opts.verbose_name)), escape(sub_obj)), []])
                 else:
                     # Display a link to the admin page.
-                    nh(deleted_objects, current_depth, ['%s: <a href="../../../../%s/%s/%s/">%s</a>' % \
+                    nh(deleted_objects, current_depth, [u'%s: <a href="../../../../%s/%s/%s/">%s</a>' % \
-                        (capfirst(related.opts.verbose_name), related.opts.app_label, related.opts.object_name.lower(), sub_obj._get_pk_val(), escape(str(sub_obj))), []])
+                        (force_unicode(capfirst(related.opts.verbose_name)), related.opts.app_label, related.opts.object_name.lower(), sub_obj._get_pk_val(), escape(sub_obj)), []])
                 _get_deleted_objects(deleted_objects, perms_needed, user, sub_obj, related.opts, current_depth+2)
             # If there were related objects, and the user doesn't have
             # permission to delete them, add the missing perm to perms_needed.
@@ -466,7 +468,7 @@ def _get_deleted_objects(deleted_objects, perms_needed, user, obj, opts, current
         opts_seen.append(related.opts)
         rel_opts_name = related.get_accessor_name()
         has_related_objs = False
-       
+
         # related.get_accessor_name() could return None for symmetrical relationships
         if rel_opts_name:
             rel_objs = getattr(obj, rel_opts_name, None)
@@ -479,17 +481,17 @@ def _get_deleted_objects(deleted_objects, perms_needed, user, obj, opts, current
                     # Don't display link to edit, because it either has no
                     # admin or is edited inline.
                     nh(deleted_objects, current_depth, [_('One or more %(fieldname)s in %(name)s: %(obj)s') % \
-                        {'fieldname': related.field.verbose_name, 'name': related.opts.verbose_name, 'obj': escape(str(sub_obj))}, []])
+                        {'fieldname': force_unicode(related.field.verbose_name), 'name': force_unicode(related.opts.verbose_name), 'obj': escape(sub_obj)}, []])
                 else:
                     # Display a link to the admin page.
                     nh(deleted_objects, current_depth, [
-                        (_('One or more %(fieldname)s in %(name)s:') % {'fieldname': related.field.verbose_name, 'name':related.opts.verbose_name}) + \
+                        (_('One or more %(fieldname)s in %(name)s:') % {'fieldname': force_unicode(related.field.verbose_name), 'name': force_unicode(related.opts.verbose_name)}) + \
-                        (' <a href="../../../../%s/%s/%s/">%s</a>' % \
+                        (u' <a href="../../../../%s/%s/%s/">%s</a>' % \
-                            (related.opts.app_label, related.opts.module_name, sub_obj._get_pk_val(), escape(str(sub_obj)))), []])
+                            (related.opts.app_label, related.opts.module_name, sub_obj._get_pk_val(), escape(sub_obj))), []])
         # If there were related objects, and the user doesn't have
         # permission to change them, add the missing perm to perms_needed.
         if related.opts.admin and has_related_objs:
-            p = '%s.%s' % (related.opts.app_label, related.opts.get_change_permission())
+            p = u'%s.%s' % (related.opts.app_label, related.opts.get_change_permission())
             if not user.has_perm(p):
                 perms_needed.add(related.opts.verbose_name)
 
@@ -505,21 +507,21 @@ def delete_stage(request, app_label, model_name, object_id):
 
     # Populate deleted_objects, a data structure of all related objects that
     # will also be deleted.
-    deleted_objects = ['%s: <a href="../../%s/">%s</a>' % (capfirst(opts.verbose_name), object_id, escape(str(obj))), []]
+    deleted_objects = [u'%s: <a href="../../%s/">%s</a>' % (force_unicode(capfirst(opts.verbose_name)), force_unicode(object_id), escape(obj)), []]
     perms_needed = set()
     _get_deleted_objects(deleted_objects, perms_needed, request.user, obj, opts, 1)
 
     if request.POST: # The user has already confirmed the deletion.
         if perms_needed:
             raise PermissionDenied
-        obj_display = str(obj)
+        obj_display = force_unicode(obj)
         obj.delete()
         LogEntry.objects.log_action(request.user.id, ContentType.objects.get_for_model(model).id, object_id, obj_display, DELETION)
-        request.user.message_set.create(message=_('The %(name)s "%(obj)s" was deleted successfully.') % {'name': opts.verbose_name, 'obj': obj_display})
+        request.user.message_set.create(message=_('The %(name)s "%(obj)s" was deleted successfully.') % {'name': force_unicode(opts.verbose_name), 'obj': obj_display})
         return HttpResponseRedirect("../../")
     extra_context = {
         "title": _("Are you sure?"),
-        "object_name": opts.verbose_name,
+        "object_name": force_unicode(opts.verbose_name),
         "object": obj,
         "deleted_objects": deleted_objects,
         "perms_lacking": perms_needed,
@@ -542,7 +544,7 @@ def history(request, app_label, model_name, object_id):
     extra_context = {
         'title': _('Change history: %s') % obj,
         'action_list': action_list,
-        'module_name': capfirst(model._meta.verbose_name_plural),
+        'module_name': force_unicode(capfirst(model._meta.verbose_name_plural)),
         'object': obj,
     }
     return render_to_response(["admin/%s/%s/object_history.html" % (app_label, model._meta.object_name.lower()),
@@ -574,7 +576,7 @@ class ChangeList(object):
         self.query = request.GET.get(SEARCH_VAR, '')
         self.query_set = self.get_query_set()
         self.get_results(request)
-        self.title = (self.is_popup and _('Select %s') % self.opts.verbose_name or _('Select %s to change') % self.opts.verbose_name)
+        self.title = (self.is_popup and _('Select %s') % force_unicode(self.opts.verbose_name) or _('Select %s to change') % force_unicode(self.opts.verbose_name))
         self.filter_specs, self.has_filters = self.get_filters(request)
         self.pk_attname = self.lookup_opts.pk.attname
 
@@ -602,7 +604,7 @@ class ChangeList(object):
                 del p[k]
             elif v is not None:
                 p[k] = v
-        return '?' + '&amp;'.join(['%s=%s' % (k, v) for k, v in p.items()]).replace(' ', '%20')
+        return '?' + '&amp;'.join([u'%s=%s' % (k, v) for k, v in p.items()]).replace(' ', '%20')
 
     def get_results(self, request):
         paginator = ObjectPaginator(self.query_set, self.lookup_opts.admin.list_per_page)
@@ -688,6 +690,12 @@ class ChangeList(object):
         for i in (ALL_VAR, ORDER_VAR, ORDER_TYPE_VAR, SEARCH_VAR, IS_POPUP_VAR):
             if i in lookup_params:
                 del lookup_params[i]
+        for key, value in lookup_params.items():
+            if not isinstance(key, str):
+                # 'key' will be used as a keyword argument later, so Python
+                # requires it to be a string.
+                del lookup_params[key]
+                lookup_params[smart_str(key)] = value
 
         # Apply lookup parameters from the query string.
         qs = qs.filter(**lookup_params)

django/contrib/auth/forms.py

@@ -4,7 +4,7 @@ from django.contrib.sites.models import Site
 from django.template import Context, loader
 from django.core import validators
 from django import oldforms
-from django.utils.translation import gettext as _
+from django.utils.translation import ugettext as _
 
 class UserCreationForm(oldforms.Manipulator):
     "A form that creates a user, with no privileges, from the given username and password."

django/contrib/auth/management.py

@@ -7,13 +7,13 @@ from django.db.models import get_models, signals
 from django.contrib.auth import models as auth_app
 
 def _get_permission_codename(action, opts):
-    return '%s_%s' % (action, opts.object_name.lower())
+    return u'%s_%s' % (action, opts.object_name.lower())
 
 def _get_all_permissions(opts):
     "Returns (codename, name) for all permissions in the given opts."
     perms = []
     for action in ('add', 'change', 'delete'):
-        perms.append((_get_permission_codename(action, opts), 'Can %s %s' % (action, opts.verbose_name)))
+        perms.append((_get_permission_codename(action, opts), u'Can %s %s' % (action, opts.verbose_name_raw)))
     return perms + list(opts.permissions)
 
 def create_permissions(app, created_models, verbosity):

django/contrib/auth/models.py

@@ -2,8 +2,10 @@ from django.core import validators
 from django.core.exceptions import ImproperlyConfigured
 from django.db import backend, connection, models
 from django.contrib.contenttypes.models import ContentType
-from django.utils.translation import gettext_lazy as _
+from django.utils.encoding import smart_str
+from django.utils.translation import ugettext_lazy as _
 import datetime
+import urllib
 
 try:
     set
@@ -18,16 +20,16 @@ def check_password(raw_password, enc_password):
     algo, salt, hsh = enc_password.split('$')
     if algo == 'md5':
         import md5
-        return hsh == md5.new(salt+raw_password).hexdigest()
+        return hsh == md5.new(smart_str(salt + raw_password)).hexdigest()
     elif algo == 'sha1':
         import sha
-        return hsh == sha.new(salt+raw_password).hexdigest()
+        return hsh == sha.new(smart_str(salt + raw_password)).hexdigest()
     elif algo == 'crypt':
         try:
             import crypt
         except ImportError:
             raise ValueError, "Crypt password algorithm not supported in this environment."
-        return hsh == crypt.crypt(raw_password, salt)
+        return hsh == crypt.crypt(smart_str(raw_password), smart_str(salt))
     raise ValueError, "Got unknown password algorithm type in password."
 
 class SiteProfileNotAvailable(Exception):
@@ -56,8 +58,8 @@ class Permission(models.Model):
         unique_together = (('content_type', 'codename'),)
         ordering = ('content_type', 'codename')
 
-    def __str__(self):
+    def __unicode__(self):
-        return "%s | %s | %s" % (self.content_type.app_label, self.content_type, self.name)
+        return u"%s | %s | %s" % (self.content_type.app_label, self.content_type, self.name)
 
 class Group(models.Model):
     """Groups are a generic way of categorizing users to apply permissions, or some other label, to those users. A user can belong to any number of groups.
@@ -77,7 +79,7 @@ class Group(models.Model):
     class Admin:
         search_fields = ('name',)
 
-    def __str__(self):
+    def __unicode__(self):
         return self.name
 
 class UserManager(models.Manager):
@@ -133,11 +135,11 @@ class User(models.Model):
         list_filter = ('is_staff', 'is_superuser')
         search_fields = ('username', 'first_name', 'last_name', 'email')
 
-    def __str__(self):
+    def __unicode__(self):
         return self.username
 
     def get_absolute_url(self):
-        return "/users/%s/" % self.username
+        return "/users/%s/" % urllib.quote(smart_str(self.username))
 
     def is_anonymous(self):
         "Always returns False. This is a way of comparing User objects to anonymous users."
@@ -150,14 +152,14 @@ class User(models.Model):
 
     def get_full_name(self):
         "Returns the first_name plus the last_name, with a space in between."
-        full_name = '%s %s' % (self.first_name, self.last_name)
+        full_name = u'%s %s' % (self.first_name, self.last_name)
         return full_name.strip()
 
     def set_password(self, raw_password):
         import sha, random
         algo = 'sha1'
         salt = sha.new(str(random.random())).hexdigest()[:5]
-        hsh = sha.new(salt+raw_password).hexdigest()
+        hsh = sha.new(salt + smart_str(raw_password)).hexdigest()
         self.password = '%s$%s$%s' % (algo, salt, hsh)
 
     def check_password(self, raw_password):
@@ -169,7 +171,7 @@ class User(models.Model):
         # algorithm or salt.
         if '$' not in self.password:
             import md5
-            is_correct = (self.password == md5.new(raw_password).hexdigest())
+            is_correct = (self.password == md5.new(smart_str(raw_password)).hexdigest())
             if is_correct:
                 # Convert the password to the new, more secure format.
                 self.set_password(raw_password)
@@ -209,7 +211,7 @@ class User(models.Model):
 
     def get_all_permissions(self):
         if not hasattr(self, '_perm_cache'):
-            self._perm_cache = set(["%s.%s" % (p.content_type.app_label, p.codename) for p in self.user_permissions.select_related()])
+            self._perm_cache = set([u"%s.%s" % (p.content_type.app_label, p.codename) for p in self.user_permissions.select_related()])
             self._perm_cache.update(self.get_group_permissions())
         return self._perm_cache
 
@@ -271,7 +273,7 @@ class Message(models.Model):
     user = models.ForeignKey(User)
     message = models.TextField(_('message'))
 
-    def __str__(self):
+    def __unicode__(self):
         return self.message
 
 class AnonymousUser(object):
@@ -281,9 +283,12 @@ class AnonymousUser(object):
     def __init__(self):
         pass
 
-    def __str__(self):
+    def __unicode__(self):
         return 'AnonymousUser'
 
+    def __str__(self):
+        return unicode(self).encode('utf-8')
+
     def __eq__(self, other):
         return isinstance(other, self.__class__)
 

django/contrib/auth/views.py

@@ -7,6 +7,7 @@ from django.contrib.sites.models import Site
 from django.http import HttpResponseRedirect
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth import REDIRECT_FIELD_NAME
+from django.utils.translation import ugettext as _
 
 def login(request, template_name='registration/login.html'):
     "Displays the login form and handles the login action."

django/contrib/comments/feeds.py

@@ -11,7 +11,7 @@ class LatestFreeCommentsFeed(Feed):
     def title(self):
         if not hasattr(self, '_site'):
             self._site = Site.objects.get_current()
-        return "%s comments" % self._site.name
+        return u"%s comments" % self._site.name
 
     def link(self):
         if not hasattr(self, '_site'):
@@ -21,7 +21,7 @@ class LatestFreeCommentsFeed(Feed):
     def description(self):
         if not hasattr(self, '_site'):
             self._site = Site.objects.get_current()
-        return "Latest comments on %s" % self._site.name
+        return u"Latest comments on %s" % self._site.name
 
     def get_query_set(self):
         return self.comments_class.objects.filter(site__pk=settings.SITE_ID, is_public=True)

django/contrib/comments/models.py

@@ -2,7 +2,7 @@ from django.db import models
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.sites.models import Site
 from django.contrib.auth.models import User
-from django.utils.translation import gettext_lazy as _
+from django.utils.translation import ugettext_lazy as _
 from django.conf import settings
 import datetime
 

django/contrib/comments/views/comments.py

@@ -11,7 +11,8 @@ from django.contrib.auth.forms import AuthenticationForm
 from django.http import HttpResponseRedirect
 from django.utils.text import normalize_newlines
 from django.conf import settings
-from django.utils.translation import ngettext
+from django.utils.translation import ungettext, ugettext as _
+from django.utils.encoding import smart_unicode
 import base64, datetime
 
 COMMENTS_PER_PAGE = 20
@@ -108,7 +109,7 @@ class PublicCommentManipulator(AuthenticationForm):
         # If the commentor has posted fewer than COMMENTS_FIRST_FEW comments,
         # send the comment to the managers.
         if self.user_cache.comment_set.count() <= settings.COMMENTS_FIRST_FEW:
-            message = ngettext('This comment was posted by a user who has posted fewer than %(count)s comment:\n\n%(text)s',
+            message = ungettext('This comment was posted by a user who has posted fewer than %(count)s comment:\n\n%(text)s',
                 'This comment was posted by a user who has posted fewer than %(count)s comments:\n\n%(text)s', settings.COMMENTS_FIRST_FEW) % \
                 {'count': settings.COMMENTS_FIRST_FEW, 'text': c.get_as_text()}
             mail_managers("Comment posted by rookie user", message)
@@ -248,7 +249,7 @@ def post_comment(request):
         # If the IP is banned, mail the admins, do NOT save the comment, and
         # serve up the "Thanks for posting" page as if the comment WAS posted.
         if request.META['REMOTE_ADDR'] in settings.BANNED_IPS:
-            mail_admins("Banned IP attempted to post comment", str(request.POST) + "\n\n" + str(request.META))
+            mail_admins("Banned IP attempted to post comment", smart_unicode(request.POST) + "\n\n" + str(request.META))
         else:
             manipulator.do_html2python(new_data)
             comment = manipulator.save(new_data)
@@ -312,7 +313,7 @@ def post_free_comment(request):
         # serve up the "Thanks for posting" page as if the comment WAS posted.
         if request.META['REMOTE_ADDR'] in settings.BANNED_IPS:
             from django.core.mail import mail_admins
-            mail_admins("Practical joker", str(request.POST) + "\n\n" + str(request.META))
+            mail_admins("Practical joker", smart_unicode(request.POST) + "\n\n" + str(request.META))
         else:
             manipulator.do_html2python(new_data)
             comment = manipulator.save(new_data)

django/contrib/comments/views/karma.py

@@ -2,6 +2,7 @@ from django.http import Http404
 from django.shortcuts import render_to_response
 from django.template import RequestContext
 from django.contrib.comments.models import Comment, KarmaScore
+from django.utils.translation import ugettext as _
 
 def vote(request, comment_id, vote):
     """

django/contrib/contenttypes/generic.py

@@ -49,7 +49,7 @@ class GenericForeignKey(object):
         
     def __get__(self, instance, instance_type=None):
         if instance is None:
-            raise AttributeError, "%s must be accessed via instance" % self.name
+            raise AttributeError, u"%s must be accessed via instance" % self.name
 
         try:
             return getattr(instance, self.cache_attr)
@@ -66,7 +66,7 @@ class GenericForeignKey(object):
 
     def __set__(self, instance, value):
         if instance is None:
-            raise AttributeError, "%s must be accessed via instance" % self.related.opts.object_name
+            raise AttributeError, u"%s must be accessed via instance" % self.related.opts.object_name
 
         ct = None
         fk = None

django/contrib/contenttypes/management.py

@@ -4,6 +4,7 @@ Creates content types for all installed models.
 
 from django.dispatch import dispatcher
 from django.db.models import get_apps, get_models, signals
+from django.utils.encoding import smart_unicode
 
 def create_contenttypes(app, created_models, verbosity=2):
     from django.contrib.contenttypes.models import ContentType
@@ -17,7 +18,7 @@ def create_contenttypes(app, created_models, verbosity=2):
             ContentType.objects.get(app_label=opts.app_label,
                 model=opts.object_name.lower())
         except ContentType.DoesNotExist:
-            ct = ContentType(name=str(opts.verbose_name),
+            ct = ContentType(name=smart_unicode(opts.verbose_name_raw),
                 app_label=opts.app_label, model=opts.object_name.lower())
             ct.save()
             if verbosity >= 2:

django/contrib/contenttypes/models.py

@@ -1,5 +1,6 @@
 from django.db import models
-from django.utils.translation import gettext_lazy as _
+from django.utils.translation import ugettext_lazy as _
+from django.utils.encoding import smart_unicode
 
 CONTENT_TYPE_CACHE = {}
 class ContentTypeManager(models.Manager):
@@ -13,13 +14,13 @@ class ContentTypeManager(models.Manager):
         try:
             ct = CONTENT_TYPE_CACHE[key]
         except KeyError:
-            # The str() is needed around opts.verbose_name because it's a
+            # The smart_unicode() is needed around opts.verbose_name_raw because it might
-            # django.utils.functional.__proxy__ object.
+            # be a django.utils.functional.__proxy__ object.
             ct, created = self.model._default_manager.get_or_create(app_label=key[0],
-                model=key[1], defaults={'name': str(opts.verbose_name)})
+                model=key[1], defaults={'name': smart_unicode(opts.verbose_name_raw)})
             CONTENT_TYPE_CACHE[key] = ct
         return ct
-        
+
     def clear_cache(self):
         """
         Clear out the content-type cache. This needs to happen during database
@@ -42,7 +43,7 @@ class ContentType(models.Model):
         ordering = ('name',)
         unique_together = (('app_label', 'model'),)
 
-    def __str__(self):
+    def __unicode__(self):
         return self.name
 
     def model_class(self):

django/contrib/databrowse/datastructures.py

@@ -7,6 +7,7 @@ from django.db import models
 from django.utils import dateformat
 from django.utils.text import capfirst
 from django.utils.translation import get_date_formats
+from django.utils.encoding import smart_unicode, smart_str, iri_to_uri
 
 EMPTY_VALUE = '(None)'
 
@@ -19,7 +20,7 @@ class EasyModel(object):
         self.verbose_name_plural = model._meta.verbose_name_plural
 
     def __repr__(self):
-        return '<EasyModel for %s>' % self.model._meta.object_name
+        return '<EasyModel for %s>' % smart_str(self.model._meta.object_name)
 
     def model_databrowse(self):
         "Returns the ModelDatabrowse class for this model."
@@ -54,7 +55,7 @@ class EasyField(object):
         self.model, self.field = easy_model, field
 
     def __repr__(self):
-        return '<EasyField for %s.%s>' % (self.model.model._meta.object_name, self.field.name)
+        return smart_str(u'<EasyField for %s.%s>' % (self.model.model._meta.object_name, self.field.name))
 
     def choices(self):
         for value, label in self.field.choices:
@@ -72,29 +73,32 @@ class EasyChoice(object):
         self.value, self.label = value, label
 
     def __repr__(self):
-        return '<EasyChoice for %s.%s>' % (self.model.model._meta.object_name, self.field.name)
+        return smart_str(u'<EasyChoice for %s.%s>' % (self.model.model._meta.object_name, self.field.name))
 
     def url(self):
-        return '%s%s/%s/%s/%s/' % (self.model.site.root_url, self.model.model._meta.app_label, self.model.model._meta.module_name, self.field.field.name, self.value)
+        return '%s%s/%s/%s/%s/' % (self.model.site.root_url, self.model.model._meta.app_label, self.model.model._meta.module_name, self.field.field.name, iri_to_uri(self.value))
 
 class EasyInstance(object):
     def __init__(self, easy_model, instance):
         self.model, self.instance = easy_model, instance
 
     def __repr__(self):
-        return '<EasyInstance for %s (%s)>' % (self.model.model._meta.object_name, self.instance._get_pk_val())
+        return smart_str(u'<EasyInstance for %s (%s)>' % (self.model.model._meta.object_name, self.instance._get_pk_val()))
+
+    def __unicode__(self):
+        val = smart_unicode(self.instance)
+        if len(val) > 30:
+            return val[:30] + u'...'
+        return val
 
     def __str__(self):
-        val = str(self.instance)
+        return self.__unicode__().encode('utf-8')
-        if len(val) > 30:
-            return val[:30] + '...'
-        return val
 
     def pk(self):
         return self.instance._get_pk_val()
 
     def url(self):
-        return '%s%s/%s/objects/%s/' % (self.model.site.root_url, self.model.model._meta.app_label, self.model.model._meta.module_name, self.pk())
+        return '%s%s/%s/objects/%s/' % (self.model.site.root_url, self.model.model._meta.app_label, self.model.model._meta.module_name, iri_to_uri(self.pk()))
 
     def fields(self):
         """
@@ -126,7 +130,7 @@ class EasyInstanceField(object):
         self.raw_value = getattr(instance.instance, field.name)
 
     def __repr__(self):
-        return '<EasyInstanceField for %s.%s>' % (self.model.model._meta.object_name, self.field.name)
+        return smart_str(u'<EasyInstanceField for %s.%s>' % (self.model.model._meta.object_name, self.field.name))
 
     def values(self):
         """
@@ -175,18 +179,18 @@ class EasyInstanceField(object):
             if self.field.rel.to in self.model.model_list:
                 lst = []
                 for value in self.values():
-                    url = '%s%s/%s/objects/%s/' % (self.model.site.root_url, m.model._meta.app_label, m.model._meta.module_name, value._get_pk_val())
+                    url = '%s%s/%s/objects/%s/' % (self.model.site.root_url, m.model._meta.app_label, m.model._meta.module_name, iri_to_uri(value._get_pk_val()))
-                    lst.append((str(value), url))
+                    lst.append((smart_unicode(value), url))
             else:
                 lst = [(value, None) for value in self.values()]
         elif self.field.choices:
             lst = []
             for value in self.values():
-                url = '%s%s/%s/fields/%s/%s/' % (self.model.site.root_url, self.model.model._meta.app_label, self.model.model._meta.module_name, self.field.name, self.raw_value)
+                url = '%s%s/%s/fields/%s/%s/' % (self.model.site.root_url, self.model.model._meta.app_label, self.model.model._meta.module_name, self.field.name, iri_to_uri(self.raw_value))
                 lst.append((value, url))
         elif isinstance(self.field, models.URLField):
             val = self.values()[0]
-            lst = [(val, val)]
+            lst = [(val, iri_to_uri(val))]
         else:
             lst = [(self.values()[0], None)]
         return lst

django/contrib/databrowse/plugins/calendars.py

@@ -6,6 +6,7 @@ from django.shortcuts import render_to_response
 from django.utils.text import capfirst
 from django.utils.translation import get_date_formats
 from django.views.generic import date_based
+from django.utils.encoding import force_unicode
 import datetime
 import time
 
@@ -27,13 +28,13 @@ class CalendarPlugin(DatabrowsePlugin):
     def model_index_html(self, request, model, site):
         fields = self.field_dict(model)
         if not fields:
-            return ''
+            return u''
-        return '<p class="filter"><strong>View calendar by:</strong> %s</p>' % \
+        return u'<p class="filter"><strong>View calendar by:</strong> %s</p>' % \
-            ', '.join(['<a href="calendars/%s/">%s</a>' % (f.name, capfirst(f.verbose_name)) for f in fields.values()])
+            u', '.join(['<a href="calendars/%s/">%s</a>' % (f.name, force_unicode(capfirst(f.verbose_name))) for f in fields.values()])
 
     def urls(self, plugin_name, easy_instance_field):
         if isinstance(easy_instance_field.field, models.DateField):
-            return ['%s%s/%s/%s/%s/%s/' % (easy_instance_field.model.url(),
+            return [u'%s%s/%s/%s/%s/%s/' % (easy_instance_field.model.url(),
                 plugin_name, easy_instance_field.field.name,
                 easy_instance_field.raw_value.year,
                 easy_instance_field.raw_value.strftime('%b').lower(),

django/contrib/databrowse/plugins/fieldchoices.py

@@ -4,9 +4,11 @@ from django.contrib.databrowse.datastructures import EasyModel
 from django.contrib.databrowse.sites import DatabrowsePlugin
 from django.shortcuts import render_to_response
 from django.utils.text import capfirst
+from django.utils.encoding import smart_str, force_unicode
 from django.views.generic import date_based
 import datetime
 import time
+import urllib
 
 class FieldChoicePlugin(DatabrowsePlugin):
     def __init__(self, field_filter=None):
@@ -29,15 +31,15 @@ class FieldChoicePlugin(DatabrowsePlugin):
     def model_index_html(self, request, model, site):
         fields = self.field_dict(model)
         if not fields:
-            return ''
+            return u''
-        return '<p class="filter"><strong>View by:</strong> %s</p>' % \
+        return u'<p class="filter"><strong>View by:</strong> %s</p>' % \
-            ', '.join(['<a href="fields/%s/">%s</a>' % (f.name, capfirst(f.verbose_name)) for f in fields.values()])
+            u', '.join(['<a href="fields/%s/">%s</a>' % (f.name, force_unicode(capfirst(f.verbose_name))) for f in fields.values()])
 
     def urls(self, plugin_name, easy_instance_field):
         if easy_instance_field.field in self.field_dict(easy_instance_field.model.model).values():
-            return ['%s%s/%s/%s/' % (easy_instance_field.model.url(),
+            return [u'%s%s/%s/%s/' % (easy_instance_field.model.url(),
                 plugin_name, easy_instance_field.field.name,
-                easy_instance_field.raw_value)]
+                urllib.quote(smart_str(easy_instance_field.raw_value)))]
 
     def model_view(self, request, model_databrowse, url):
         self.model, self.site = model_databrowse.model, model_databrowse.site

django/contrib/databrowse/sites.py

@@ -60,7 +60,7 @@ class ModelDatabrowse(object):
 
     def main_view(self, request):
         easy_model = EasyModel(self.site, self.model)
-        html_snippets = '\n'.join([p.model_index_html(request, self.model, self.site) for p in self.plugins.values()])
+        html_snippets = u'\n'.join([p.model_index_html(request, self.model, self.site) for p in self.plugins.values()])
         return render_to_response('databrowse/model_detail.html', {
             'model': easy_model,
             'root_url': self.site.root_url,

django/contrib/databrowse/templates/databrowse/fieldchoice_list.html

@@ -10,7 +10,7 @@
 
 <ul class="objectlist">
 {% for object in object_list %}
-<li class="{% cycle odd,even %}"><a href="{{ object }}/">{{ object|escape }}</a></li>
+<li class="{% cycle odd,even %}"><a href="{{ object|iriencode }}/">{{ object|escape }}</a></li>
 {% endfor %}
 </ul>
 

django/contrib/flatpages/models.py

@@ -1,7 +1,7 @@
 from django.core import validators
 from django.db import models
 from django.contrib.sites.models import Site
-from django.utils.translation import gettext_lazy as _
+from django.utils.translation import ugettext_lazy as _
 
 class FlatPage(models.Model):
     url = models.CharField(_('URL'), maxlength=100, validator_list=[validators.isAlphaNumericURL], db_index=True,
@@ -26,8 +26,8 @@ class FlatPage(models.Model):
         list_filter = ('sites',)
         search_fields = ('url', 'title')
 
-    def __str__(self):
+    def __unicode__(self):
-        return "%s -- %s" % (self.url, self.title)
+        return u"%s -- %s" % (self.url, self.title)
 
     def get_absolute_url(self):
         return self.url

django/contrib/humanize/templatetags/humanize.py

@@ -1,5 +1,5 @@
-from django.utils.translation import ngettext
+from django.utils.translation import ungettext, ugettext as _
-from django.utils.translation import gettext_lazy as _
+from django.utils.encoding import force_unicode
 from django import template
 import re
 
@@ -16,8 +16,8 @@ def ordinal(value):
         return value
     t = (_('th'), _('st'), _('nd'), _('rd'), _('th'), _('th'), _('th'), _('th'), _('th'), _('th'))
     if value % 100 in (11, 12, 13): # special case
-        return "%d%s" % (value, t[0])
+        return u"%d%s" % (value, t[0])
-    return '%d%s' % (value, t[value % 10])
+    return u'%d%s' % (value, t[value % 10])
 register.filter(ordinal)
 
 def intcomma(value):
@@ -25,8 +25,8 @@ def intcomma(value):
     Converts an integer to a string containing commas every three digits.
     For example, 3000 becomes '3,000' and 45000 becomes '45,000'.
     """
-    orig = str(value)
+    orig = force_unicode(value)
-    new = re.sub("^(-?\d+)(\d{3})", '\g<1>,\g<2>', str(value))
+    new = re.sub("^(-?\d+)(\d{3})", '\g<1>,\g<2>', orig)
     if orig == new:
         return new
     else:
@@ -44,13 +44,13 @@ def intword(value):
         return value
     if value < 1000000000:
         new_value = value / 1000000.0
-        return ngettext('%(value).1f million', '%(value).1f million', new_value) % {'value': new_value}
+        return ungettext('%(value).1f million', '%(value).1f million', new_value) % {'value': new_value}
     if value < 1000000000000:
         new_value = value / 1000000000.0
-        return ngettext('%(value).1f billion', '%(value).1f billion', new_value) % {'value': new_value}
+        return ungettext('%(value).1f billion', '%(value).1f billion', new_value) % {'value': new_value}
     if value < 1000000000000000:
         new_value = value / 1000000000000.0
-        return ngettext('%(value).1f trillion', '%(value).1f trillion', new_value) % {'value': new_value}
+        return ungettext('%(value).1f trillion', '%(value).1f trillion', new_value) % {'value': new_value}
     return value
 register.filter(intword)
 

django/contrib/localflavor/au/forms.py

@@ -5,7 +5,7 @@ Australian-specific Form helpers
 from django.newforms import ValidationError
 from django.newforms.fields import Field, RegexField, Select, EMPTY_VALUES
 from django.newforms.util import smart_unicode
-from django.utils.translation import gettext
+from django.utils.translation import ugettext
 import re
 
 PHONE_DIGITS_RE = re.compile(r'^(\d{10})$')
@@ -15,14 +15,14 @@ class AUPostCodeField(RegexField):
     def __init__(self, *args, **kwargs):
         super(AUPostCodeField, self).__init__(r'^\d{4}$',
             max_length=None, min_length=None,
-            error_message=gettext(u'Enter a 4 digit post code.'),
+            error_message=ugettext('Enter a 4 digit post code.'),
-            *args, **kwargs)
+                    *args, **kwargs)
 
 class AUPhoneNumberField(Field):
     """Australian phone number field."""
     def clean(self, value):
-        """Validate a phone number. Strips parentheses, whitespace and
+        """
-        hyphens.
+        Validate a phone number. Strips parentheses, whitespace and hyphens.
         """
         super(AUPhoneNumberField, self).clean(value)
         if value in EMPTY_VALUES:
@@ -39,5 +39,5 @@ class AUStateSelect(Select):
     choices.
     """
     def __init__(self, attrs=None):
-        from au_states import STATE_CHOICES # relative import
+        from au_states import STATE_CHOICES
         super(AUStateSelect, self).__init__(attrs, choices=STATE_CHOICES)

django/contrib/localflavor/br/forms.py

@@ -6,7 +6,7 @@ BR-specific Form helpers
 from django.newforms import ValidationError
 from django.newforms.fields import Field, RegexField, CharField, Select, EMPTY_VALUES
 from django.utils.encoding import smart_unicode
-from django.utils.translation import gettext
+from django.utils.translation import ugettext
 import re
 
 phone_digits_re = re.compile(r'^(\d{2})[-\.]?(\d{4})[-\.]?(\d{4})$')
@@ -15,8 +15,8 @@ class BRZipCodeField(RegexField):
     def __init__(self, *args, **kwargs):
         super(BRZipCodeField, self).__init__(r'^\d{5}-\d{3}$',
             max_length=None, min_length=None,
-            error_message=gettext('Enter a zip code in the format XXXXX-XXX.'),
+            error_message=ugettext('Enter a zip code in the format XXXXX-XXX.'),
-            *args, **kwargs)
+                    *args, **kwargs)
 
 class BRPhoneNumberField(Field):
     def clean(self, value):
@@ -27,7 +27,7 @@ class BRPhoneNumberField(Field):
         m = phone_digits_re.search(value)
         if m:
             return u'%s-%s-%s' % (m.group(1), m.group(2), m.group(3))
-        raise ValidationError(gettext(u'Phone numbers must be in XX-XXXX-XXXX format.'))
+        raise ValidationError(ugettext('Phone numbers must be in XX-XXXX-XXXX format.'))
 
 class BRStateSelect(Select):
     """
@@ -35,7 +35,7 @@ class BRStateSelect(Select):
     as its choices.
     """
     def __init__(self, attrs=None):
-        from br_states import STATE_CHOICES # relative import
+        from br_states import STATE_CHOICES
         super(BRStateSelect, self).__init__(attrs, choices=STATE_CHOICES)
 
 
@@ -69,9 +69,9 @@ class BRCPFField(CharField):
         try:
             int(value)
         except ValueError:
-            raise ValidationError(gettext("This field requires only numbers."))
+            raise ValidationError(ugettext("This field requires only numbers."))
         if len(value) != 11:
-            raise ValidationError(gettext("This field requires at most 11 digits or 14 characters."))
+            raise ValidationError(ugettext("This field requires at most 11 digits or 14 characters."))
         orig_dv = value[-2:]
 
         new_1dv = sum([i * int(value[idx]) for idx, i in enumerate(range(10, 1, -1))])
@@ -81,7 +81,7 @@ class BRCPFField(CharField):
         new_2dv = DV_maker(new_2dv % 11)
         value = value[:-1] + str(new_2dv)
         if value[-2:] != orig_dv:
-            raise ValidationError(gettext("Invalid CPF number."))
+            raise ValidationError(ugettext("Invalid CPF number."))
 
         return orig_value
 
@@ -103,7 +103,7 @@ class BRCNPJField(Field):
             raise ValidationError("This field requires only numbers.")
         if len(value) != 14:
             raise ValidationError(
-                gettext("This field requires at least 14 digits"))
+                ugettext("This field requires at least 14 digits"))
         orig_dv = value[-2:]
 
         new_1dv = sum([i * int(value[idx]) for idx, i in enumerate(range(5, 1, -1) + range(9, 1, -1))])
@@ -113,7 +113,7 @@ class BRCNPJField(Field):
         new_2dv = DV_maker(new_2dv % 11)
         value = value[:-1] + str(new_2dv)
         if value[-2:] != orig_dv:
-            raise ValidationError(gettext("Invalid CNPJ number."))
+            raise ValidationError(ugettext("Invalid CNPJ number."))
 
         return orig_value
 

django/contrib/localflavor/ch/ch_states.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*
-from django.utils.translation import gettext_lazy as _
+from django.utils.translation import ugettext_lazy as _
 
 STATE_CHOICES = (
     ('AG', _('Aargau')),

django/contrib/localflavor/ch/forms.py

@@ -5,7 +5,7 @@ Swiss-specific Form helpers
 from django.newforms import ValidationError
 from django.newforms.fields import Field, RegexField, Select, EMPTY_VALUES
 from django.utils.encoding import smart_unicode
-from django.utils.translation import gettext
+from django.utils.translation import ugettext
 import re
 
 id_re = re.compile(r"^(?P<idnumber>\w{8})(?P<pos9>(\d{1}|<))(?P<checksum>\d{1})$")
@@ -15,7 +15,7 @@ class CHZipCodeField(RegexField):
     def __init__(self, *args, **kwargs):
         super(CHZipCodeField, self).__init__(r'^\d{4}$',
         max_length=None, min_length=None,
-        error_message=gettext('Enter a zip code in the format XXXX.'),
+        error_message=ugettext('Enter a zip code in the format XXXX.'),
         *args, **kwargs)
 
 class CHPhoneNumberField(Field):
@@ -87,7 +87,7 @@ class CHIdentityCardNumberField(Field):
 
     def clean(self, value):
         super(CHIdentityCardNumberField, self).clean(value)
-        error_msg = gettext('Enter a valid Swiss identity or passport card number in X1234567<0 or 1234567890 format.')
+        error_msg = ugettext('Enter a valid Swiss identity or passport card number in X1234567<0 or 1234567890 format.')
         if value in EMPTY_VALUES:
             return u''
 

django/contrib/localflavor/cl/forms.py

@@ -4,7 +4,8 @@ Chile specific form helpers.
 
 from django.newforms import ValidationError
 from django.newforms.fields import RegexField, EMPTY_VALUES
-from django.utils.translation import gettext
+from django.utils.translation import ugettext
+from django.utils.encoding import smart_unicode
 
 class CLRutField(RegexField):
     """
@@ -18,12 +19,12 @@ class CLRutField(RegexField):
         if 'strict' in kwargs:
             del kwargs['strict']
             super(CLRutField, self).__init__(r'^(\d{1,2}\.)?\d{3}\.\d{3}-[\dkK]$',
-                error_message=gettext('Enter valid a Chilean RUT. The format is XX.XXX.XXX-X.'),
+                error_message=ugettext('Enter valid a Chilean RUT. The format is XX.XXX.XXX-X.'),
-                *args, **kwargs)
+                        *args, **kwargs)
         else:
             # In non-strict mode, accept RUTs that validate but do not exist in
             # the real world.
-            super(CLRutField, self).__init__(r'^[\d\.]{1,11}-?[\dkK]$', error_message=gettext(u'Enter valid a Chilean RUT'), *args, **kwargs)
+            super(CLRutField, self).__init__(r'^[\d\.]{1,11}-?[\dkK]$', error_message=ugettext('Enter valid a Chilean RUT'), *args, **kwargs)
 
     def clean(self, value):
         """
@@ -49,14 +50,14 @@ class CLRutField(RegexField):
             multi += 1
             if multi == 8:
                 multi = 2
-        return '0123456789K0'[11 - suma % 11]
+        return u'0123456789K0'[11 - suma % 11]
 
     def _canonify(self, rut):
         """
         Turns the RUT into one normalized format. Returns a (rut, verifier)
         tuple.
         """
-        rut = str(rut).replace(' ', '').replace('.', '').replace('-', '')
+        rut = smart_unicode(rut).replace(' ', '').replace('.', '').replace('-', '')
         return rut[:-1], rut[-1]
 
     def _format(self, code, verifier=None):
@@ -74,5 +75,5 @@ class CLRutField(RegexField):
             else:
                 new_dot = pos - 3
             code = code[:new_dot] + '.' + code[new_dot:]
-        return '%s-%s' % (code, verifier)
+        return u'%s-%s' % (code, verifier)
 

django/contrib/localflavor/de/de_states.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*
-from django.utils.translation import gettext_lazy as _
+from django.utils.translation import ugettext_lazy as _
 
 STATE_CHOICES = (
     ('BW', _('Baden-Wuerttemberg')),

django/contrib/localflavor/de/forms.py

@@ -4,7 +4,7 @@ DE-specific Form helpers
 
 from django.newforms import ValidationError
 from django.newforms.fields import Field, RegexField, Select, EMPTY_VALUES
-from django.utils.translation import gettext
+from django.utils.translation import ugettext
 import re
 
 id_re = re.compile(r"^(?P<residence>\d{10})(?P<origin>\w{1,3})[-\ ]?(?P<birthday>\d{7})[-\ ]?(?P<validity>\d{7})[-\ ]?(?P<checksum>\d{1})$")
@@ -13,15 +13,15 @@ class DEZipCodeField(RegexField):
     def __init__(self, *args, **kwargs):
         super(DEZipCodeField, self).__init__(r'^\d{5}$',
             max_length=None, min_length=None,
-            error_message=gettext(u'Enter a zip code in the format XXXXX.'),
+            error_message=ugettext('Enter a zip code in the format XXXXX.'),
-            *args, **kwargs)
+                    *args, **kwargs)
 
 class DEStateSelect(Select):
     """
     A Select widget that uses a list of DE states as its choices.
     """
     def __init__(self, attrs=None):
-        from de_states import STATE_CHOICES # relative import
+        from de_states import STATE_CHOICES
         super(DEStateSelect, self).__init__(attrs, choices=STATE_CHOICES)
 
 class DEIdentityCardNumberField(Field):
@@ -57,7 +57,7 @@ class DEIdentityCardNumberField(Field):
 
     def clean(self, value):
         super(DEIdentityCardNumberField, self).clean(value)
-        error_msg = gettext(u'Enter a valid German identity card number in XXXXXXXXXXX-XXXXXXX-XXXXXXX-X format.')
+        error_msg = ugettext('Enter a valid German identity card number in XXXXXXXXXXX-XXXXXXX-XXXXXXX-X format.')
         if value in EMPTY_VALUES:
             return u''
         match = re.match(id_re, value)
@@ -71,7 +71,7 @@ class DEIdentityCardNumberField(Field):
         if residence == '0000000000' or birthday == '0000000' or validity == '0000000':
             raise ValidationError(error_msg)
 
-        all_digits = "%s%s%s%s" % (residence, birthday, validity, checksum)
+        all_digits = u"%s%s%s%s" % (residence, birthday, validity, checksum)
         if not self.has_valid_checksum(residence) or not self.has_valid_checksum(birthday) or \
             not self.has_valid_checksum(validity) or not self.has_valid_checksum(all_digits):
                 raise ValidationError(error_msg)

django/contrib/localflavor/fi/forms.py

@@ -5,21 +5,21 @@ FI-specific Form helpers
 import re
 from django.newforms import ValidationError
 from django.newforms.fields import Field, RegexField, Select, EMPTY_VALUES
-from django.utils.translation import gettext
+from django.utils.translation import ugettext
 
 class FIZipCodeField(RegexField):
     def __init__(self, *args, **kwargs):
         super(FIZipCodeField, self).__init__(r'^\d{5}$',
             max_length=None, min_length=None,
-            error_message=gettext(u'Enter a zip code in the format XXXXX.'),
+            error_message=ugettext('Enter a zip code in the format XXXXX.'),
-            *args, **kwargs)
+                    *args, **kwargs)
 
 class FIMunicipalitySelect(Select):
     """
     A Select widget that uses a list of Finnish municipalities as its choices.
     """
     def __init__(self, attrs=None):
-        from fi_municipalities import MUNICIPALITY_CHOICES # relative import
+        from fi_municipalities import MUNICIPALITY_CHOICES
         super(FIMunicipalitySelect, self).__init__(attrs, choices=MUNICIPALITY_CHOICES)
 
 class FISocialSecurityNumber(Field):
@@ -37,9 +37,9 @@ class FISocialSecurityNumber(Field):
             (?P<serial>(\d{3}))
             (?P<checksum>[%s])$""" % checkmarks, value, re.VERBOSE | re.IGNORECASE)
         if not result:
-            raise ValidationError(gettext(u'Enter a valid Finnish social security number.'))
+            raise ValidationError(ugettext('Enter a valid Finnish social security number.'))
         gd = result.groupdict()
         checksum = int(gd['date'] + gd['serial'])
         if checkmarks[checksum % len(checkmarks)] == gd['checksum'].upper():
             return u'%s' % value.upper()
-        raise ValidationError(gettext(u'Enter a valid Finnish social security number.'))
+        raise ValidationError(ugettext('Enter a valid Finnish social security number.'))

django/contrib/localflavor/fr/forms.py

@@ -5,7 +5,7 @@ FR-specific Form helpers
 from django.newforms import ValidationError
 from django.newforms.fields import Field, RegexField, Select, EMPTY_VALUES
 from django.utils.encoding import smart_unicode
-from django.utils.translation import gettext
+from django.utils.translation import ugettext
 import re
 
 phone_digits_re = re.compile(r'^0\d(\s|\.)?(\d{2}(\s|\.)?){3}\d{2}$')
@@ -14,8 +14,8 @@ class FRZipCodeField(RegexField):
     def __init__(self, *args, **kwargs):
         super(FRZipCodeField, self).__init__(r'^\d{5}$',
             max_length=None, min_length=None,
-            error_message=gettext(u'Enter a zip code in the format XXXXX.'),
+            error_message=ugettext('Enter a zip code in the format XXXXX.'),
-            *args, **kwargs)
+                    *args, **kwargs)
 
 class FRPhoneNumberField(Field):
     """
@@ -39,6 +39,6 @@ class FRDepartmentSelect(Select):
     A Select widget that uses a list of FR departments as its choices.
     """
     def __init__(self, attrs=None):
-        from fr_department import DEPARTMENT_ASCII_CHOICES # relative import
+        from fr_department import DEPARTMENT_ASCII_CHOICES
         super(FRDepartmentSelect, self).__init__(attrs, choices=DEPARTMENT_ASCII_CHOICES)
 

django/contrib/localflavor/is_/forms.py

@@ -5,7 +5,8 @@ Iceland specific form helpers.
 from django.newforms import ValidationError
 from django.newforms.fields import RegexField, EMPTY_VALUES
 from django.newforms.widgets import Select
-from django.utils.translation import gettext
+from django.utils.translation import ugettext
+from django.utils.encoding import smart_unicode
 
 class ISIdNumberField(RegexField):
     """
@@ -13,7 +14,7 @@ class ISIdNumberField(RegexField):
     of Iceland has.
     """
     def __init__(self, *args, **kwargs):
-        error_msg = gettext(u'Enter a valid Icelandic identification number. The format is XXXXXX-XXXX.')
+        error_msg = ugettext('Enter a valid Icelandic identification number. The format is XXXXXX-XXXX.')
         kwargs['min_length'],kwargs['max_length'] = 10,11
         super(ISIdNumberField, self).__init__(r'^\d{6}(-| )?\d{4}$', error_message=error_msg, *args, **kwargs)
 
@@ -27,7 +28,7 @@ class ISIdNumberField(RegexField):
         if self._validate(value):
             return self._format(value)
         else:
-            raise ValidationError(gettext(u'The Icelandic identification number is not valid.'))
+            raise ValidationError(ugettext(u'The Icelandic identification number is not valid.'))
 
     def _canonify(self, value):
         """
@@ -48,7 +49,7 @@ class ISIdNumberField(RegexField):
         Takes in the value in canonical form and returns it in the common
         display format.
         """
-        return value[:6]+'-'+value[6:]
+        return smart_unicode(value[:6]+'-'+value[6:])
 
 class ISPhoneNumberField(RegexField):
     """

django/contrib/localflavor/it/forms.py

@@ -4,7 +4,7 @@ IT-specific Form helpers
 
 from django.newforms import ValidationError
 from django.newforms.fields import Field, RegexField, Select, EMPTY_VALUES
-from django.utils.translation import gettext
+from django.utils.translation import ugettext
 from django.utils.encoding import smart_unicode
 from django.contrib.localflavor.it.util import ssn_check_digit, vat_number_check_digit
 import re
@@ -13,15 +13,15 @@ class ITZipCodeField(RegexField):
     def __init__(self, *args, **kwargs):
         super(ITZipCodeField, self).__init__(r'^\d{5}$',
         max_length=None, min_length=None,
-        error_message=gettext(u'Enter a valid zip code.'),
+        error_message=ugettext('Enter a valid zip code.'),
-        *args, **kwargs)
+                *args, **kwargs)
 
 class ITRegionSelect(Select):
     """
     A Select widget that uses a list of IT regions as its choices.
     """
     def __init__(self, attrs=None):
-        from it_region import REGION_CHOICES # relative import
+        from it_region import REGION_CHOICES
         super(ITRegionSelect, self).__init__(attrs, choices=REGION_CHOICES)
 
 class ITProvinceSelect(Select):
@@ -29,7 +29,7 @@ class ITProvinceSelect(Select):
     A Select widget that uses a list of IT regions as its choices.
     """
     def __init__(self, attrs=None):
-        from it_province import PROVINCE_CHOICES # relative import
+        from it_province import PROVINCE_CHOICES
         super(ITProvinceSelect, self).__init__(attrs, choices=PROVINCE_CHOICES)
 
 class ITSocialSecurityNumberField(RegexField):
@@ -38,7 +38,7 @@ class ITSocialSecurityNumberField(RegexField):
     For reference see http://www.agenziaentrate.it/ and search for
     'Informazioni sulla codificazione delle persone fisiche'.
     """
-    err_msg = gettext(u'Enter a valid Social Security number.')
+    err_msg = ugettext(u'Enter a valid Social Security number.')
     def __init__(self, *args, **kwargs):
         super(ITSocialSecurityNumberField, self).__init__(r'^\w{3}\s*\w{3}\s*\w{5}\s*\w{5}$',
         max_length=None, min_length=None, error_message=self.err_msg,
@@ -65,7 +65,7 @@ class ITVatNumberField(Field):
         value = super(ITVatNumberField, self).clean(value)
         if value == u'':
             return value
-        err_msg = gettext(u'Enter a valid VAT number.')
+        err_msg = ugettext(u'Enter a valid VAT number.')
         try:
             vat_number = int(value)
         except ValueError:

django/contrib/localflavor/it/it_province.py

@@ -33,7 +33,7 @@ PROVINCE_CHOICES = (
     ('KR', 'Crotone'),
     ('CN', 'Cuneo'),
     ('EN', 'Enna'),
-#    ('FM', 'Fermo'), # active starting from 2009    
+#    ('FM', 'Fermo'), # active starting from 2009
     ('FE', 'Ferrara'),
     ('FI', 'Firenze'),
     ('FG', 'Foggia'),

django/contrib/localflavor/it/util.py

@@ -1,23 +1,27 @@
+from django.utils.encoding import smart_str, smart_unicode
+
 def ssn_check_digit(value):
     "Calculate Italian social security number check digit."
     ssn_even_chars = {
-        '0': 0, '1': 1, '2': 2, '3': 3, '4': 4, '5': 5, '6': 6, '7': 7, '8': 8, '9': 9,
+        '0': 0, '1': 1, '2': 2, '3': 3, '4': 4, '5': 5, '6': 6, '7': 7, '8': 8,
-        'A': 0, 'B': 1, 'C': 2, 'D': 3, 'E': 4, 'F': 5, 'G': 6, 'H': 7, 'I': 8, 'J': 9,
+        '9': 9, 'A': 0, 'B': 1, 'C': 2, 'D': 3, 'E': 4, 'F': 5, 'G': 6, 'H': 7,
-        'K': 10, 'L': 11, 'M': 12, 'N': 13, 'O': 14, 'P': 15, 'Q': 16, 'R': 17, 'S': 18,
+        'I': 8, 'J': 9, 'K': 10, 'L': 11, 'M': 12, 'N': 13, 'O': 14, 'P': 15,
-        'T': 19, 'U': 20, 'V': 21, 'W': 22, 'X': 23, 'Y': 24, 'Z': 25
+        'Q': 16, 'R': 17, 'S': 18, 'T': 19, 'U': 20, 'V': 21, 'W': 22, 'X': 23,
+        'Y': 24, 'Z': 25
     }
     ssn_odd_chars = {
-        '0': 1, '1': 0, '2': 5, '3': 7, '4': 9, '5': 13, '6': 15, '7': 17, '8': 19, '9': 21,
+        '0': 1, '1': 0, '2': 5, '3': 7, '4': 9, '5': 13, '6': 15, '7': 17, '8':
-        'A': 1, 'B': 0, 'C': 5, 'D': 7, 'E': 9, 'F': 13, 'G': 15, 'H': 17, 'I': 19, 'J': 21,
+        19, '9': 21, 'A': 1, 'B': 0, 'C': 5, 'D': 7, 'E': 9, 'F': 13, 'G': 15,
-        'K': 2, 'L': 4, 'M': 18, 'N': 20, 'O': 11, 'P': 3, 'Q': 6, 'R': 8, 'S': 12,
+        'H': 17, 'I': 19, 'J': 21, 'K': 2, 'L': 4, 'M': 18, 'N': 20, 'O': 11,
-        'T': 14, 'U': 16, 'V': 10, 'W': 22, 'X': 25, 'Y': 24, 'Z': 23
+        'P': 3, 'Q': 6, 'R': 8, 'S': 12, 'T': 14, 'U': 16, 'V': 10, 'W': 22,
+        'X': 25, 'Y': 24, 'Z': 23
     }
     # Chars from 'A' to 'Z'
     ssn_check_digits = [chr(x) for x in range(65, 91)]
 
     ssn = value.upper()
     total = 0
-    for i in range(0,15):
+    for i in range(0, 15):
         try:
             if i % 2 == 0:
                 total += ssn_odd_chars[ssn[i]]
@@ -30,11 +34,11 @@ def ssn_check_digit(value):
 
 def vat_number_check_digit(vat_number):
     "Calculate Italian VAT number check digit."
-    normalized_vat_number = str(vat_number).zfill(10)
+    normalized_vat_number = smart_str(vat_number).zfill(10)
     total = 0
     for i in range(0, 10, 2):
         total += int(normalized_vat_number[i])
     for i in range(1, 11, 2):
         quotient , remainder = divmod(int(normalized_vat_number[i]) * 2, 10)
         total += quotient + remainder
-    return str((10 - total % 10) % 10)
+    return smart_unicode((10 - total % 10) % 10)

django/contrib/localflavor/jp/forms.py

@@ -4,7 +4,7 @@ JP-specific Form helpers
 
 from django.core import validators
 from django.newforms import ValidationError
-from django.utils.translation import gettext
+from django.utils.translation import ugettext
 from django.newforms.fields import RegexField, Select
 
 import re
@@ -18,8 +18,8 @@ class JPPostalCodeField(RegexField):
     def __init__(self, *args, **kwargs):
         super(JPPostalCodeField, self).__init__(r'^\d{3}-\d{4}$|^\d{7}$',
             max_length=None, min_length=None,
-            error_message=gettext(u'Enter a postal code in the format XXXXXXX or XXX-XXXX.'),
+            error_message=ugettext('Enter a postal code in the format XXXXXXX or XXX-XXXX.'),
-            *args, **kwargs)
+                    *args, **kwargs)
 
     def clean(self, value):
         """

django/contrib/localflavor/jp/jp_prefectures.py

@@ -1,51 +1,51 @@
-from django.utils.translation import gettext_lazy as gettext_lazy
+from django.utils.translation import ugettext_lazy
 
 JP_PREFECTURES = (
-    ('hokkaido', gettext_lazy('Hokkaido'),),
+    ('hokkaido', ugettext_lazy('Hokkaido'),),
-    ('aomori', gettext_lazy('Aomori'),),
+    ('aomori', ugettext_lazy('Aomori'),),
-    ('iwate', gettext_lazy('Iwate'),),
+    ('iwate', ugettext_lazy('Iwate'),),
-    ('miyagi', gettext_lazy('Miyagi'),),
+    ('miyagi', ugettext_lazy('Miyagi'),),
-    ('akita', gettext_lazy('Akita'),),
+    ('akita', ugettext_lazy('Akita'),),
-    ('yamagata', gettext_lazy('Yamagata'),),
+    ('yamagata', ugettext_lazy('Yamagata'),),
-    ('fukushima', gettext_lazy('Fukushima'),),
+    ('fukushima', ugettext_lazy('Fukushima'),),
-    ('ibaraki', gettext_lazy('Ibaraki'),),
+    ('ibaraki', ugettext_lazy('Ibaraki'),),
-    ('tochigi', gettext_lazy('Tochigi'),),
+    ('tochigi', ugettext_lazy('Tochigi'),),
-    ('gunma', gettext_lazy('Gunma'),),
+    ('gunma', ugettext_lazy('Gunma'),),
-    ('saitama', gettext_lazy('Saitama'),),
+    ('saitama', ugettext_lazy('Saitama'),),
-    ('chiba', gettext_lazy('Chiba'),),
+    ('chiba', ugettext_lazy('Chiba'),),
-    ('tokyo', gettext_lazy('Tokyo'),),
+    ('tokyo', ugettext_lazy('Tokyo'),),
-    ('kanagawa', gettext_lazy('Kanagawa'),),
+    ('kanagawa', ugettext_lazy('Kanagawa'),),
-    ('yamanashi', gettext_lazy('Yamanashi'),),
+    ('yamanashi', ugettext_lazy('Yamanashi'),),
-    ('nagano', gettext_lazy('Nagano'),),
+    ('nagano', ugettext_lazy('Nagano'),),
-    ('niigata', gettext_lazy('Niigata'),),
+    ('niigata', ugettext_lazy('Niigata'),),
-    ('toyama', gettext_lazy('Toyama'),),
+    ('toyama', ugettext_lazy('Toyama'),),
-    ('ishikawa', gettext_lazy('Ishikawa'),),
+    ('ishikawa', ugettext_lazy('Ishikawa'),),
-    ('fukui', gettext_lazy('Fukui'),),
+    ('fukui', ugettext_lazy('Fukui'),),
-    ('gifu', gettext_lazy('Gifu'),),
+    ('gifu', ugettext_lazy('Gifu'),),
-    ('shizuoka', gettext_lazy('Shizuoka'),),
+    ('shizuoka', ugettext_lazy('Shizuoka'),),
-    ('aichi', gettext_lazy('Aichi'),),
+    ('aichi', ugettext_lazy('Aichi'),),
-    ('mie', gettext_lazy('Mie'),),
+    ('mie', ugettext_lazy('Mie'),),
-    ('shiga', gettext_lazy('Shiga'),),
+    ('shiga', ugettext_lazy('Shiga'),),
-    ('kyoto', gettext_lazy('Kyoto'),),
+    ('kyoto', ugettext_lazy('Kyoto'),),
-    ('osaka', gettext_lazy('Osaka'),),
+    ('osaka', ugettext_lazy('Osaka'),),
-    ('hyogo', gettext_lazy('Hyogo'),),
+    ('hyogo', ugettext_lazy('Hyogo'),),
-    ('nara', gettext_lazy('Nara'),),
+    ('nara', ugettext_lazy('Nara'),),
-    ('wakayama', gettext_lazy('Wakayama'),),
+    ('wakayama', ugettext_lazy('Wakayama'),),
-    ('tottori', gettext_lazy('Tottori'),),
+    ('tottori', ugettext_lazy('Tottori'),),
-    ('shimane', gettext_lazy('Shimane'),),
+    ('shimane', ugettext_lazy('Shimane'),),
-    ('okayama', gettext_lazy('Okayama'),),
+    ('okayama', ugettext_lazy('Okayama'),),
-    ('hiroshima', gettext_lazy('Hiroshima'),),
+    ('hiroshima', ugettext_lazy('Hiroshima'),),
-    ('yamaguchi', gettext_lazy('Yamaguchi'),),
+    ('yamaguchi', ugettext_lazy('Yamaguchi'),),
-    ('tokushima', gettext_lazy('Tokushima'),),
+    ('tokushima', ugettext_lazy('Tokushima'),),
-    ('kagawa', gettext_lazy('Kagawa'),),
+    ('kagawa', ugettext_lazy('Kagawa'),),
-    ('ehime', gettext_lazy('Ehime'),),
+    ('ehime', ugettext_lazy('Ehime'),),
-    ('kochi', gettext_lazy('Kochi'),),
+    ('kochi', ugettext_lazy('Kochi'),),
-    ('fukuoka', gettext_lazy('Fukuoka'),),
+    ('fukuoka', ugettext_lazy('Fukuoka'),),
-    ('saga', gettext_lazy('Saga'),),
+    ('saga', ugettext_lazy('Saga'),),
-    ('nagasaki', gettext_lazy('Nagasaki'),),
+    ('nagasaki', ugettext_lazy('Nagasaki'),),
-    ('kumamoto', gettext_lazy('Kumamoto'),),
+    ('kumamoto', ugettext_lazy('Kumamoto'),),
-    ('oita', gettext_lazy('Oita'),),
+    ('oita', ugettext_lazy('Oita'),),
-    ('miyazaki', gettext_lazy('Miyazaki'),),
+    ('miyazaki', ugettext_lazy('Miyazaki'),),
-    ('kagoshima', gettext_lazy('Kagoshima'),),
+    ('kagoshima', ugettext_lazy('Kagoshima'),),
-    ('okinawa', gettext_lazy('Okinawa'),),
+    ('okinawa', ugettext_lazy('Okinawa'),),
 )

django/contrib/localflavor/no/forms.py

@@ -6,14 +6,14 @@ Norwegian-specific Form helpers
 import re, datetime
 from django.newforms import ValidationError
 from django.newforms.fields import Field, RegexField, Select, EMPTY_VALUES
-from django.utils.translation import gettext
+from django.utils.translation import ugettext
 
 class NOZipCodeField(RegexField):
     def __init__(self, *args, **kwargs):
         super(NOZipCodeField, self).__init__(r'^\d{4}$',
             max_length=None, min_length=None,
-            error_message=gettext(u'Enter a zip code in the format XXXX.'),
+            error_message=ugettext('Enter a zip code in the format XXXX.'),
-            *args, **kwargs)
+                    *args, **kwargs)
 
 class NOMunicipalitySelect(Select):
     """
@@ -33,7 +33,7 @@ class NOSocialSecurityNumber(Field):
         if value in EMPTY_VALUES:
             return u''
 
-        msg = gettext(u'Enter a valid Norwegian social security number.')
+        msg = ugettext(u'Enter a valid Norwegian social security number.')
         if not re.match(r'^\d{11}$', value):
             raise ValidationError(msg)
 
@@ -60,7 +60,7 @@ class NOSocialSecurityNumber(Field):
             self.gender = 'F'
         else:
             self.gender = 'M'
-        
+
         digits = map(int, list(value))
         weight_1 = [3, 7, 6, 1, 8, 9, 4, 5, 2, 1, 0]
         weight_2 = [5, 4, 3, 2, 7, 6, 5, 4, 3, 2, 1]
@@ -74,4 +74,4 @@ class NOSocialSecurityNumber(Field):
             raise ValidationError(msg)
 
         return value
-    
+

django/contrib/localflavor/no/no_municipalities.py

@@ -1,4 +1,4 @@
-# -*- coding: iso-8859-1 -*-
+# -*- coding: utf-8 -*-
 """
 An alphabetical list of Norwegian municipalities (fylker) fro use as `choices`
 in a formfield.
@@ -15,18 +15,18 @@ MUNICIPALITY_CHOICES = (
     ('hedmark', u'Hedmark'),
     ('hordaland', u'Hordaland'),
     ('janmayen', u'Jan Mayen'),
-    ('moreogromsdal', u'Møre og Romsdal'),
+    ('moreogromsdal', u'Møre og Romsdal'),
-    ('nordtrondelag', u'Nord-Trøndelag'),
+    ('nordtrondelag', u'Nord-Trøndelag'),
     ('nordland', u'Nordland'),
     ('oppland', u'Oppland'),
     ('oslo', u'Oslo'),
     ('rogaland', u'Rogaland'),
     ('sognogfjordane', u'Sogn og Fjordane'),
     ('svalbard', u'Svalbard'),
-    ('sortrondelag', u'Sør-Trøndelag'),
+    ('sortrondelag', u'Sør-Trøndelag'),
     ('telemark', u'Telemark'),
     ('troms', u'Troms'),
     ('vestagder', u'Vest-Agder'),
     ('vestfold', u'Vestfold'),
-    ('ostfold', u'Østfold')
+    ('ostfold', u'Østfold')
 )

django/contrib/localflavor/uk/forms.py

@@ -3,7 +3,7 @@ UK-specific Form helpers
 """
 
 from django.newforms.fields import RegexField
-from django.utils.translation import gettext
+from django.utils.translation import ugettext
 
 class UKPostcodeField(RegexField):
     """
@@ -15,5 +15,5 @@ class UKPostcodeField(RegexField):
     def __init__(self, *args, **kwargs):
         super(UKPostcodeField, self).__init__(r'^(GIR 0AA|[A-PR-UWYZ]([0-9]{1,2}|([A-HIK-Y][0-9](|[0-9]|[ABEHMNPRVWXY]))|[0-9][A-HJKSTUW]) [0-9][ABD-HJLNP-UW-Z]{2})$',
             max_length=None, min_length=None,
-            error_message=gettext(u'Enter a postcode. A space is required between the two postcode parts.'),
+            error_message=ugettext(u'Enter a postcode. A space is required between the two postcode parts.'),
             *args, **kwargs)

django/contrib/localflavor/us/forms.py

@@ -5,7 +5,7 @@ USA-specific Form helpers
 from django.newforms import ValidationError
 from django.newforms.fields import Field, RegexField, Select, EMPTY_VALUES
 from django.utils.encoding import smart_unicode
-from django.utils.translation import gettext
+from django.utils.translation import ugettext
 import re
 
 phone_digits_re = re.compile(r'^(?:1-?)?(\d{3})[-\.]?(\d{3})[-\.]?(\d{4})$')
@@ -15,8 +15,8 @@ class USZipCodeField(RegexField):
     def __init__(self, *args, **kwargs):
         super(USZipCodeField, self).__init__(r'^\d{5}(?:-\d{4})?$',
             max_length=None, min_length=None,
-            error_message=gettext(u'Enter a zip code in the format XXXXX or XXXXX-XXXX.'),
+            error_message=ugettext('Enter a zip code in the format XXXXX or XXXXX-XXXX.'),
-            *args, **kwargs)
+                    *args, **kwargs)
 
 class USPhoneNumberField(Field):
     def clean(self, value):
@@ -38,17 +38,17 @@ class USSocialSecurityNumberField(Field):
         * Conforms to the XXX-XX-XXXX format.
         * No group consists entirely of zeroes.
         * The leading group is not "666" (block "666" will never be allocated).
-        * The number is not in the promotional block 987-65-4320 through 987-65-4329,
+        * The number is not in the promotional block 987-65-4320 through
-          which are permanently invalid.
+          987-65-4329, which are permanently invalid.
         * The number is not one known to be invalid due to otherwise widespread
-          promotional use or distribution (e.g., the Woolworth's number or the 1962
+          promotional use or distribution (e.g., the Woolworth's number or the
-          promotional number).
+          1962 promotional number).
     """
     def clean(self, value):
         super(USSocialSecurityNumberField, self).clean(value)
         if value in EMPTY_VALUES:
             return u''
-        msg = gettext(u'Enter a valid U.S. Social Security number in XXX-XX-XXXX format.')
+        msg = ugettext('Enter a valid U.S. Social Security number in XXX-XX-XXXX format.')
         match = re.match(ssn_re, value)
         if not match:
             raise ValidationError(msg)
@@ -75,7 +75,7 @@ class USStateField(Field):
     abbreviation for the given state.
     """
     def clean(self, value):
-        from us_states import STATES_NORMALIZED # relative import
+        from us_states import STATES_NORMALIZED
         super(USStateField, self).clean(value)
         if value in EMPTY_VALUES:
             return u''
@@ -95,5 +95,5 @@ class USStateSelect(Select):
     A Select widget that uses a list of U.S. states/territories as its choices.
     """
     def __init__(self, attrs=None):
-        from us_states import STATE_CHOICES # relative import
+        from us_states import STATE_CHOICES
         super(USStateSelect, self).__init__(attrs, choices=STATE_CHOICES)

django/contrib/markup/templatetags/markup.py

@@ -16,6 +16,7 @@ silently fail and return the un-marked-up text.
 
 from django import template
 from django.conf import settings
+from django.utils.encoding import smart_str, force_unicode
 
 register = template.Library()
 
@@ -25,9 +26,9 @@ def textile(value):
     except ImportError:
         if settings.DEBUG:
             raise template.TemplateSyntaxError, "Error in {% textile %} filter: The Python textile library isn't installed."
-        return value
+        return force_unicode(value)
     else:
-        return textile.textile(value, encoding=settings.DEFAULT_CHARSET, output=settings.DEFAULT_CHARSET)
+        return force_unicode(textile.textile(smart_str(value), encoding='utf-8', output='utf-8'))
 
 def markdown(value):
     try:
@@ -35,9 +36,9 @@ def markdown(value):
     except ImportError:
         if settings.DEBUG:
             raise template.TemplateSyntaxError, "Error in {% markdown %} filter: The Python markdown library isn't installed."
-        return value
+        return force_unicode(value)
     else:
-        return markdown.markdown(value)
+        return force_unicode(markdown.markdown(smart_str(value)))
 
 def restructuredtext(value):
     try:
@@ -45,11 +46,11 @@ def restructuredtext(value):
     except ImportError:
         if settings.DEBUG:
             raise template.TemplateSyntaxError, "Error in {% restructuredtext %} filter: The Python docutils library isn't installed."
-        return value
+        return force_unicode(value)
     else:
         docutils_settings = getattr(settings, "RESTRUCTUREDTEXT_FILTER_SETTINGS", {})
-        parts = publish_parts(source=value, writer_name="html4css1", settings_overrides=docutils_settings)
+        parts = publish_parts(source=smart_str(value), writer_name="html4css1", settings_overrides=docutils_settings)
-        return parts["fragment"]
+        return force_unicode(parts["fragment"])
 
 register.filter(textile)
 register.filter(markdown)

django/contrib/redirects/models.py

@@ -1,6 +1,6 @@
 from django.db import models
 from django.contrib.sites.models import Site
-from django.utils.translation import gettext_lazy as _
+from django.utils.translation import ugettext_lazy as _
 
 class Redirect(models.Model):
     site = models.ForeignKey(Site, radio_admin=models.VERTICAL)
@@ -20,5 +20,5 @@ class Redirect(models.Model):
         list_filter = ('site',)
         search_fields = ('old_path', 'new_path')
 
-    def __str__(self):
+    def __unicode__(self):
-        return "%s ---> %s" % (self.old_path, self.new_path)
+        return u"%s ---> %s" % (self.old_path, self.new_path)

django/contrib/sessions/models.py

@@ -1,7 +1,7 @@
 import base64, md5, random, sys, datetime, os, time
 import cPickle as pickle
 from django.db import models
-from django.utils.translation import gettext_lazy as _
+from django.utils.translation import ugettext_lazy as _
 from django.conf import settings
 
 class SessionManager(models.Manager):

django/contrib/sitemaps/views.py

@@ -2,6 +2,7 @@ from django.http import HttpResponse, Http404
 from django.template import loader
 from django.contrib.sites.models import Site
 from django.core import urlresolvers
+from django.utils.encoding import smart_str
 
 def index(request, sitemaps):
     current_site = Site.objects.get_current()
@@ -26,5 +27,5 @@ def sitemap(request, sitemaps, section=None):
             urls.extend(site().get_urls())
         else:
             urls.extend(site.get_urls())
-    xml = loader.render_to_string('sitemap.xml', {'urlset': urls})
+    xml = smart_str(loader.render_to_string('sitemap.xml', {'urlset': urls}))
     return HttpResponse(xml, mimetype='application/xml')

django/contrib/sites/models.py

@@ -1,5 +1,5 @@
 from django.db import models
-from django.utils.translation import gettext_lazy as _
+from django.utils.translation import ugettext_lazy as _
 
 class SiteManager(models.Manager):
     def get_current(self):
@@ -19,5 +19,5 @@ class Site(models.Model):
         list_display = ('domain', 'name')
         search_fields = ('domain', 'name')
 
-    def __str__(self):
+    def __unicode__(self):
         return self.domain

django/contrib/syndication/feeds.py

@@ -2,10 +2,13 @@ from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist
 from django.template import Context, loader, Template, TemplateDoesNotExist
 from django.contrib.sites.models import Site
 from django.utils import feedgenerator
+from django.utils.encoding import smart_unicode
 from django.conf import settings
 
 def add_domain(domain, url):
     if not url.startswith('http://'):
+        # 'url' must already be ASCII and URL-quoted, so no need for encoding
+        # conversions here.
         url = u'http://%s%s' % (domain, url)
     return url
 
@@ -97,9 +100,9 @@ class Feed(object):
             enc_url = self.__get_dynamic_attr('item_enclosure_url', item)
             if enc_url:
                 enc = feedgenerator.Enclosure(
-                    url = enc_url.decode('utf-8'),
+                    url = smart_unicode(enc_url),
-                    length = str(self.__get_dynamic_attr('item_enclosure_length', item)).decode('utf-8'),
+                    length = smart_unicode(self.__get_dynamic_attr('item_enclosure_length', item)),
-                    mime_type = self.__get_dynamic_attr('item_enclosure_mime_type', item).decode('utf-8'),
+                    mime_type = smart_unicode(self.__get_dynamic_attr('item_enclosure_mime_type', item))
                 )
             author_name = self.__get_dynamic_attr('item_author_name', item)
             if author_name is not None:
@@ -108,9 +111,9 @@ class Feed(object):
             else:
                 author_email = author_link = None
             feed.add_item(
-                title = title_tmp.render(Context({'obj': item, 'site': current_site})).decode('utf-8'),
+                title = title_tmp.render(Context({'obj': item, 'site': current_site})),
                 link = link,
-                description = description_tmp.render(Context({'obj': item, 'site': current_site})).decode('utf-8'),
+                description = description_tmp.render(Context({'obj': item, 'site': current_site})),
                 unique_id = link,
                 enclosure = enc,
                 pubdate = self.__get_dynamic_attr('item_pubdate', item),

django/contrib/webdesign/lorem_ipsum.py

@@ -5,8 +5,40 @@ Utility functions for generating "lorem ipsum" Latin text.
 import random
 
 COMMON_P = 'Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.'
-WORDS = ('exercitationem', 'perferendis', 'perspiciatis', 'laborum', 'eveniet', 'sunt', 'iure', 'nam', 'nobis', 'eum', 'cum', 'officiis', 'excepturi', 'odio', 'consectetur', 'quasi', 'aut', 'quisquam', 'vel', 'eligendi', 'itaque', 'non', 'odit', 'tempore', 'quaerat', 'dignissimos', 'facilis', 'neque', 'nihil', 'expedita', 'vitae', 'vero', 'ipsum', 'nisi', 'animi', 'cumque', 'pariatur', 'velit', 'modi', 'natus', 'iusto', 'eaque', 'sequi', 'illo', 'sed', 'ex', 'et', 'voluptatibus', 'tempora', 'veritatis', 'ratione', 'assumenda', 'incidunt', 'nostrum', 'placeat', 'aliquid', 'fuga', 'provident', 'praesentium', 'rem', 'necessitatibus', 'suscipit', 'adipisci', 'quidem', 'possimus', 'voluptas', 'debitis', 'sint', 'accusantium', 'unde', 'sapiente', 'voluptate', 'qui', 'aspernatur', 'laudantium', 'soluta', 'amet', 'quo', 'aliquam', 'saepe', 'culpa', 'libero', 'ipsa', 'dicta', 'reiciendis', 'nesciunt', 'doloribus', 'autem', 'impedit', 'minima', 'maiores', 'repudiandae', 'ipsam', 'obcaecati', 'ullam', 'enim', 'totam', 'delectus', 'ducimus', 'quis', 'voluptates', 'dolores', 'molestiae', 'harum', 'dolorem', 'quia', 'voluptatem', 'molestias', 'magni', 'distinctio', 'omnis', 'illum', 'dolorum', 'voluptatum', 'ea', 'quas', 'quam', 'corporis', 'quae', 'blanditiis', 'atque', 'deserunt', 'laboriosam', 'earum', 'consequuntur', 'hic', 'cupiditate', 'quibusdam', 'accusamus', 'ut', 'rerum', 'error', 'minus', 'eius', 'ab', 'ad', 'nemo', 'fugit', 'officia', 'at', 'in', 'id', 'quos', 'reprehenderit', 'numquam', 'iste', 'fugiat', 'sit', 'inventore', 'beatae', 'repellendus', 'magnam', 'recusandae', 'quod', 'explicabo', 'doloremque', 'aperiam', 'consequatur', 'asperiores', 'commodi', 'optio', 'dolor', 'labore', 'temporibus', 'repellat', 'veniam', 'architecto', 'est', 'esse', 'mollitia', 'nulla', 'a', 'similique', 'eos', 'alias', 'dolore', 'tenetur', 'deleniti', 'porro', 'facere', 'maxime', 'corrupti')
+
-COMMON_WORDS = ('lorem', 'ipsum', 'dolor', 'sit', 'amet', 'consectetur', 'adipisicing', 'elit', 'sed', 'do', 'eiusmod', 'tempor', 'incididunt', 'ut', 'labore', 'et', 'dolore', 'magna', 'aliqua')
+WORDS = ('exercitationem', 'perferendis', 'perspiciatis', 'laborum', 'eveniet',
+        'sunt', 'iure', 'nam', 'nobis', 'eum', 'cum', 'officiis', 'excepturi',
+        'odio', 'consectetur', 'quasi', 'aut', 'quisquam', 'vel', 'eligendi',
+        'itaque', 'non', 'odit', 'tempore', 'quaerat', 'dignissimos',
+        'facilis', 'neque', 'nihil', 'expedita', 'vitae', 'vero', 'ipsum',
+        'nisi', 'animi', 'cumque', 'pariatur', 'velit', 'modi', 'natus',
+        'iusto', 'eaque', 'sequi', 'illo', 'sed', 'ex', 'et', 'voluptatibus',
+        'tempora', 'veritatis', 'ratione', 'assumenda', 'incidunt', 'nostrum',
+        'placeat', 'aliquid', 'fuga', 'provident', 'praesentium', 'rem',
+        'necessitatibus', 'suscipit', 'adipisci', 'quidem', 'possimus',
+        'voluptas', 'debitis', 'sint', 'accusantium', 'unde', 'sapiente',
+        'voluptate', 'qui', 'aspernatur', 'laudantium', 'soluta', 'amet',
+        'quo', 'aliquam', 'saepe', 'culpa', 'libero', 'ipsa', 'dicta',
+        'reiciendis', 'nesciunt', 'doloribus', 'autem', 'impedit', 'minima',
+        'maiores', 'repudiandae', 'ipsam', 'obcaecati', 'ullam', 'enim',
+        'totam', 'delectus', 'ducimus', 'quis', 'voluptates', 'dolores',
+        'molestiae', 'harum', 'dolorem', 'quia', 'voluptatem', 'molestias',
+        'magni', 'distinctio', 'omnis', 'illum', 'dolorum', 'voluptatum', 'ea',
+        'quas', 'quam', 'corporis', 'quae', 'blanditiis', 'atque', 'deserunt',
+        'laboriosam', 'earum', 'consequuntur', 'hic', 'cupiditate',
+        'quibusdam', 'accusamus', 'ut', 'rerum', 'error', 'minus', 'eius',
+        'ab', 'ad', 'nemo', 'fugit', 'officia', 'at', 'in', 'id', 'quos',
+        'reprehenderit', 'numquam', 'iste', 'fugiat', 'sit', 'inventore',
+        'beatae', 'repellendus', 'magnam', 'recusandae', 'quod', 'explicabo',
+        'doloremque', 'aperiam', 'consequatur', 'asperiores', 'commodi',
+        'optio', 'dolor', 'labore', 'temporibus', 'repellat', 'veniam',
+        'architecto', 'est', 'esse', 'mollitia', 'nulla', 'a', 'similique',
+        'eos', 'alias', 'dolore', 'tenetur', 'deleniti', 'porro', 'facere',
+        'maxime', 'corrupti')
+
+COMMON_WORDS = ('lorem', 'ipsum', 'dolor', 'sit', 'amet', 'consectetur',
+        'adipisicing', 'elit', 'sed', 'do', 'eiusmod', 'tempor', 'incididunt',
+        'ut', 'labore', 'et', 'dolore', 'magna', 'aliqua')
 
 def sentence():
     """
@@ -17,10 +49,10 @@ def sentence():
     """
     # Determine the number of comma-separated sections and number of words in
     # each section for this sentence.
-    sections = [' '.join(random.sample(WORDS, random.randint(3, 12))) for i in range(random.randint(1, 5))]
+    sections = [u' '.join(random.sample(WORDS, random.randint(3, 12))) for i in range(random.randint(1, 5))]
-    s = ', '.join(sections)
+    s = u', '.join(sections)
     # Convert to sentence case and add end punctuation.
-    return '%s%s%s' % (s[0].upper(), s[1:], random.choice('?.'))
+    return u'%s%s%s' % (s[0].upper(), s[1:], random.choice('?.'))
 
 def paragraph():
     """
@@ -28,7 +60,7 @@ def paragraph():
 
     The paragraph consists of between 1 and 4 sentences, inclusive.
     """
-    return ' '.join([sentence() for i in range(random.randint(1, 4))])
+    return u' '.join([sentence() for i in range(random.randint(1, 4))])
 
 def paragraphs(count, common=True):
     """
@@ -66,4 +98,4 @@ def words(count, common=True):
             word_list += random.sample(WORDS, c)
     else:
         word_list = word_list[:count]
-    return ' '.join(word_list)
+    return u' '.join(word_list)

django/contrib/webdesign/templatetags/webdesign.py

@@ -18,7 +18,7 @@ class LoremNode(template.Node):
             paras = paragraphs(count, common=self.common)
         if self.method == 'p':
             paras = ['<p>%s</p>' % p for p in paras]
-        return '\n\n'.join(paras)
+        return u'\n\n'.join(paras)
 
 #@register.tag
 def lorem(parser, token):

django/contrib/webdesign/tests.py

@@ -2,7 +2,7 @@
 
 r"""
 >>> words(7)
-'lorem ipsum dolor sit amet consectetur adipisicing'
+u'lorem ipsum dolor sit amet consectetur adipisicing'
 
 >>> paragraphs(1)
 ['Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.']
@@ -14,4 +14,4 @@ import datetime
 
 if __name__ == '__main__':
     import doctest
-    doctest.testmod()
+    doctest.testmod()

django/core/handlers/modpython.py

@@ -49,7 +49,7 @@ class ModPythonRequest(http.HttpRequest):
         if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
             self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
         else:
-            self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
+            self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
 
     def _get_request(self):
         if not hasattr(self, '_request'):
@@ -58,7 +58,7 @@ class ModPythonRequest(http.HttpRequest):
 
     def _get_get(self):
         if not hasattr(self, '_get'):
-            self._get = http.QueryDict(self._req.args)
+            self._get = http.QueryDict(self._req.args, encoding=self._encoding)
         return self._get
 
     def _set_get(self, get):
@@ -160,7 +160,7 @@ class ModPythonHandler(BaseHandler):
         req.content_type = response['Content-Type']
         for key, value in response.headers.items():
             if key != 'Content-Type':
-                req.headers_out[key] = value
+                req.headers_out[str(key)] = str(value)
         for c in response.cookies.values():
             req.headers_out.add('Set-Cookie', c.output(header=''))
         req.status = response.status_code

django/core/handlers/wsgi.py

@@ -113,9 +113,9 @@ class WSGIRequest(http.HttpRequest):
                 header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
                 self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
             else:
-                self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
+                self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
         else:
-            self._post, self._files = http.QueryDict(''), datastructures.MultiValueDict()
+            self._post, self._files = http.QueryDict('', encoding=self._encoding), datastructures.MultiValueDict()
 
     def _get_request(self):
         if not hasattr(self, '_request'):
@@ -125,7 +125,7 @@ class WSGIRequest(http.HttpRequest):
     def _get_get(self):
         if not hasattr(self, '_get'):
             # The WSGI spec says 'QUERY_STRING' may be absent.
-            self._get = http.QueryDict(self.environ.get('QUERY_STRING', ''))
+            self._get = http.QueryDict(self.environ.get('QUERY_STRING', ''), encoding=self._encoding)
         return self._get
 
     def _set_get(self, get):
@@ -200,8 +200,8 @@ class WSGIHandler(BaseHandler):
         except KeyError:
             status_text = 'UNKNOWN STATUS CODE'
         status = '%s %s' % (response.status_code, status_text)
-        response_headers = response.headers.items()
+        response_headers = [(str(k), str(v)) for k, v in response.headers.items()]
         for c in response.cookies.values():
-            response_headers.append(('Set-Cookie', c.output(header='')))
+            response_headers.append(('Set-Cookie', str(c.output(header=''))))
         start_response(status, response_headers)
         return response

django/core/mail.py

@@ -3,12 +3,13 @@ Tools for sending email.
 """
 
 from django.conf import settings
+from django.utils.encoding import smart_str, force_unicode
 from email import Charset, Encoders
 from email.MIMEText import MIMEText
 from email.MIMEMultipart import MIMEMultipart
 from email.MIMEBase import MIMEBase
 from email.Header import Header
-from email.Utils import formatdate
+from email.Utils import formatdate, parseaddr, formataddr
 import mimetypes
 import os
 import smtplib
@@ -67,8 +68,18 @@ class SafeMIMEText(MIMEText):
         "Forbids multi-line headers, to prevent header injection."
         if '\n' in val or '\r' in val:
             raise BadHeaderError, "Header values can't contain newlines (got %r for header %r)" % (val, name)
-        if name == "Subject":
+        try:
-            val = Header(val, settings.DEFAULT_CHARSET)
+            val = str(force_unicode(val))
+        except UnicodeEncodeError:
+            if name.lower() in ('to', 'from', 'cc'):
+                result = []
+                for item in val.split(', '):
+                    nm, addr = parseaddr(item)
+                    nm = str(Header(nm, settings.DEFAULT_CHARSET))
+                    result.append(formataddr((nm, str(addr))))
+                val = ', '.join(result)
+            else:
+                val = Header(force_unicode(val), settings.DEFAULT_CHARSET)
         MIMEText.__setitem__(self, name, val)
 
 class SafeMIMEMultipart(MIMEMultipart):
@@ -76,8 +87,18 @@ class SafeMIMEMultipart(MIMEMultipart):
         "Forbids multi-line headers, to prevent header injection."
         if '\n' in val or '\r' in val:
             raise BadHeaderError, "Header values can't contain newlines (got %r for header %r)" % (val, name)
-        if name == "Subject":
+        try:
-            val = Header(val, settings.DEFAULT_CHARSET)
+            val = str(force_unicode(val))
+        except UnicodeEncodeError:
+            if name.lower() in ('to', 'from', 'cc'):
+                result = []
+                for item in val.split(', '):
+                    nm, addr = parseaddr(item)
+                    nm = str(Header(nm, settings.DEFAULT_CHARSET))
+                    result.append(formataddr((nm, str(addr))))
+                val = ', '.join(result)
+            else:
+                val = Header(force_unicode(val), settings.DEFAULT_CHARSET)
         MIMEMultipart.__setitem__(self, name, val)
 
 class SMTPConnection(object):
@@ -176,6 +197,14 @@ class EmailMessage(object):
 
     def __init__(self, subject='', body='', from_email=None, to=None, bcc=None,
             connection=None, attachments=None, headers=None):
+        """
+        Initialise a single email message (which can be sent to multiple
+        recipients).
+
+        All strings used to create the message can be unicode strings (or UTF-8
+        bytestrings). The SafeMIMEText class will handle any necessary encoding
+        conversions.
+        """
         self.to = to or []
         self.bcc = bcc or []
         self.from_email = from_email or settings.DEFAULT_FROM_EMAIL
@@ -192,7 +221,7 @@ class EmailMessage(object):
 
     def message(self):
         encoding = self.encoding or settings.DEFAULT_CHARSET
-        msg = SafeMIMEText(self.body, self.content_subtype, encoding)
+        msg = SafeMIMEText(smart_str(self.body, settings.DEFAULT_CHARSET), self.content_subtype, encoding)
         if self.attachments:
             body_msg = msg
             msg = SafeMIMEMultipart(_subtype=self.multipart_subtype)

django/core/management.py

@@ -427,11 +427,11 @@ def get_custom_sql_for_model(model):
     for sql_file in sql_files:
         if os.path.exists(sql_file):
             fp = open(sql_file, 'U')
-            for statement in statements.split(fp.read()):
+            for statement in statements.split(fp.read().decode(settings.FILE_CHARSET)):
                 # Remove any comments from the file
-                statement = re.sub(r"--.*[\n\Z]", "", statement)
+                statement = re.sub(ur"--.*[\n\Z]", "", statement)
                 if statement.strip():
-                    output.append(statement + ";")
+                    output.append(statement + u";")
             fp.close()
 
     return output

django/core/serializers/__init__.py

@@ -6,7 +6,7 @@ Usage::
     >>> from django.core import serializers
     >>> json = serializers.serialize("json", some_query_set)
     >>> objects = list(serializers.deserialize("json", json))
-        
+
 To add your own serializers, use the SERIALIZATION_MODULES setting::
 
     SERIALIZATION_MODULES = {
@@ -30,19 +30,19 @@ try:
     import yaml
     BUILTIN_SERIALIZERS["yaml"] = "django.core.serializers.pyyaml"
 except ImportError:
-    pass    
+    pass
 
 _serializers = {}
-        
+
 def register_serializer(format, serializer_module):
     """Register a new serializer by passing in a module name."""
     module = __import__(serializer_module, {}, {}, [''])
     _serializers[format] = module
-    
+
 def unregister_serializer(format):
     """Unregister a given serializer"""
     del _serializers[format]
-    
+
 def get_serializer(format):
     if not _serializers:
         _load_serializers()
@@ -52,12 +52,12 @@ def get_serializer_formats():
     if not _serializers:
         _load_serializers()
     return _serializers.keys()
-    
+
 def get_deserializer(format):
     if not _serializers:
         _load_serializers()
     return _serializers[format].Deserializer
-    
+
 def serialize(format, queryset, **options):
     """
     Serialize a queryset (or any iterator that returns database objects) using
@@ -87,4 +87,4 @@ def _load_serializers():
         register_serializer(format, BUILTIN_SERIALIZERS[format])
     if hasattr(settings, "SERIALIZATION_MODULES"):
         for format in settings.SERIALIZATION_MODULES:
-            register_serializer(format, settings.SERIALIZATION_MODULES[format])
+            register_serializer(format, settings.SERIALIZATION_MODULES[format])

django/core/serializers/base.py

@@ -7,6 +7,7 @@ try:
 except ImportError:
     from StringIO import StringIO
 from django.db import models
+from django.utils.encoding import smart_str, smart_unicode
 
 class SerializationError(Exception):
     """Something bad happened during serialization."""
@@ -59,7 +60,7 @@ class Serializer(object):
             value = getattr(obj, "get_%s_url" % field.name, lambda: None)()
         else:
             value = field.flatten_data(follow=None, obj=obj).get(field.name, "")
-        return str(value)
+        return smart_unicode(value)
 
     def start_serialization(self):
         """
@@ -154,7 +155,7 @@ class DeserializedObject(object):
         self.m2m_data = m2m_data
 
     def __repr__(self):
-        return "<DeserializedObject: %s>" % str(self.object)
+        return "<DeserializedObject: %s>" % smart_str(self.object)
 
     def save(self, save_m2m=True):
         self.object.save()

django/core/serializers/python.py

@@ -7,33 +7,34 @@ other serializers.
 from django.conf import settings
 from django.core.serializers import base
 from django.db import models
+from django.utils.encoding import smart_unicode
 
 class Serializer(base.Serializer):
     """
     Serializes a QuerySet to basic Python objects.
     """
-    
+
     def start_serialization(self):
         self._current = None
         self.objects = []
-        
+
     def end_serialization(self):
         pass
-        
+
     def start_object(self, obj):
         self._current = {}
-        
+
     def end_object(self, obj):
         self.objects.append({
-            "model"  : str(obj._meta),
+            "model"  : smart_unicode(obj._meta),
-            "pk"     : str(obj._get_pk_val()),
+            "pk"     : smart_unicode(obj._get_pk_val()),
             "fields" : self._current
         })
         self._current = None
-        
+
     def handle_field(self, obj, field):
         self._current[field.name] = getattr(obj, field.name)
-        
+
     def handle_fk_field(self, obj, field):
         related = getattr(obj, field.name)
         if related is not None:
@@ -44,17 +45,17 @@ class Serializer(base.Serializer):
                 # Related to remote object via other field
                 related = getattr(related, field.rel.field_name)
         self._current[field.name] = related
-    
+
     def handle_m2m_field(self, obj, field):
         self._current[field.name] = [related._get_pk_val() for related in getattr(obj, field.name).iterator()]
-    
+
     def getvalue(self):
         return self.objects
 
 def Deserializer(object_list, **options):
     """
     Deserialize simple Python objects back into Django ORM instances.
-    
+
     It's expected that you pass the Python objects themselves (instead of a
     stream or a string) to the constructor
     """
@@ -64,36 +65,30 @@ def Deserializer(object_list, **options):
         Model = _get_model(d["model"])
         data = {Model._meta.pk.attname : Model._meta.pk.to_python(d["pk"])}
         m2m_data = {}
-        
+
         # Handle each field
         for (field_name, field_value) in d["fields"].iteritems():
-            if isinstance(field_value, unicode):
+            if isinstance(field_value, str):
-                field_value = field_value.encode(options.get("encoding", settings.DEFAULT_CHARSET))
+                field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True)
-                
+
             field = Model._meta.get_field(field_name)
-            
+
             # Handle M2M relations
             if field.rel and isinstance(field.rel, models.ManyToManyRel):
-                pks = []
                 m2m_convert = field.rel.to._meta.pk.to_python
-                for pk in field_value:
+                m2m_data[field.name] = [m2m_convert(smart_unicode(pk)) for pk in field_value]
-                    if isinstance(pk, unicode):
+
-                        pks.append(m2m_convert(pk.encode(options.get("encoding", settings.DEFAULT_CHARSET))))
-                    else:
-                        pks.append(m2m_convert(pk))
-                m2m_data[field.name] = pks
-                
             # Handle FK fields
             elif field.rel and isinstance(field.rel, models.ManyToOneRel):
                 if field_value:
                     data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
                 else:
                     data[field.attname] = None
-                    
+
             # Handle all other fields
             else:
                 data[field.name] = field.to_python(field_value)
-                
+
         yield base.DeserializedObject(Model(**data), m2m_data)
 
 def _get_model(model_identifier):
@@ -105,5 +100,5 @@ def _get_model(model_identifier):
     except TypeError:
         Model = None
     if Model is None:
-        raise base.DeserializationError("Invalid model identifier: '%s'" % model_identifier)
+        raise base.DeserializationError(u"Invalid model identifier: '%s'" % model_identifier)
     return Model

django/core/serializers/pyyaml.py

@@ -21,7 +21,7 @@ class Serializer(PythonSerializer):
         self.options.pop('stream', None)
         self.options.pop('fields', None)
         yaml.dump(self.objects, self.stream, **self.options)
-        
+
     def getvalue(self):
         return self.stream.getvalue()
 
@@ -35,4 +35,4 @@ def Deserializer(stream_or_string, **options):
         stream = stream_or_string
     for obj in PythonDeserializer(yaml.load(stream)):
         yield obj
-        
+

django/core/serializers/xml_serializer.py

@@ -6,13 +6,14 @@ from django.conf import settings
 from django.core.serializers import base
 from django.db import models
 from django.utils.xmlutils import SimplerXMLGenerator
+from django.utils.encoding import smart_unicode
 from xml.dom import pulldom
 
 class Serializer(base.Serializer):
     """
     Serializes a QuerySet to XML.
     """
-    
+
     def indent(self, level):
         if self.options.get('indent', None) is not None:
             self.xml.ignorableWhitespace('\n' + ' ' * self.options.get('indent', None) * level)
@@ -24,7 +25,7 @@ class Serializer(base.Serializer):
         self.xml = SimplerXMLGenerator(self.stream, self.options.get("encoding", settings.DEFAULT_CHARSET))
         self.xml.startDocument()
         self.xml.startElement("django-objects", {"version" : "1.0"})
-        
+
     def end_serialization(self):
         """
         End serialization -- end the document.
@@ -32,27 +33,27 @@ class Serializer(base.Serializer):
         self.indent(0)
         self.xml.endElement("django-objects")
         self.xml.endDocument()
-        
+
     def start_object(self, obj):
         """
         Called as each object is handled.
         """
         if not hasattr(obj, "_meta"):
             raise base.SerializationError("Non-model object (%s) encountered during serialization" % type(obj))
-            
+
         self.indent(1)
         self.xml.startElement("object", {
-            "pk"    : str(obj._get_pk_val()),
+            "pk"    : smart_unicode(obj._get_pk_val()),
-            "model" : str(obj._meta),
+            "model" : smart_unicode(obj._meta),
         })
-        
+
     def end_object(self, obj):
         """
         Called after handling all fields for an object.
         """
         self.indent(1)
         self.xml.endElement("object")
-        
+
     def handle_field(self, obj, field):
         """
         Called to handle each field on an object (except for ForeignKeys and
@@ -63,17 +64,17 @@ class Serializer(base.Serializer):
             "name" : field.name,
             "type" : field.get_internal_type()
         })
-        
+
         # Get a "string version" of the object's data (this is handled by the
-        # serializer base class). 
+        # serializer base class).
         if getattr(obj, field.name) is not None:
             value = self.get_string_value(obj, field)
-            self.xml.characters(str(value))
+            self.xml.characters(smart_unicode(value))
         else:
             self.xml.addQuickElement("None")
 
         self.xml.endElement("field")
-        
+
     def handle_fk_field(self, obj, field):
         """
         Called to handle a ForeignKey (we need to treat them slightly
@@ -88,11 +89,11 @@ class Serializer(base.Serializer):
             else:
                 # Related to remote object via other field
                 related = getattr(related, field.rel.field_name)
-            self.xml.characters(str(related))
+            self.xml.characters(smart_unicode(related))
         else:
             self.xml.addQuickElement("None")
         self.xml.endElement("field")
-    
+
     def handle_m2m_field(self, obj, field):
         """
         Called to handle a ManyToManyField. Related objects are only
@@ -101,9 +102,9 @@ class Serializer(base.Serializer):
         """
         self._start_relational_field(field)
         for relobj in getattr(obj, field.name).iterator():
-            self.xml.addQuickElement("object", attrs={"pk" : str(relobj._get_pk_val())})
+            self.xml.addQuickElement("object", attrs={"pk" : smart_unicode(relobj._get_pk_val())})
         self.xml.endElement("field")
-        
+
     def _start_relational_field(self, field):
         """
         Helper to output the <field> element for relational fields
@@ -112,33 +113,33 @@ class Serializer(base.Serializer):
         self.xml.startElement("field", {
             "name" : field.name,
             "rel"  : field.rel.__class__.__name__,
-            "to"   : str(field.rel.to._meta),
+            "to"   : smart_unicode(field.rel.to._meta),
         })
-        
+
 class Deserializer(base.Deserializer):
     """
     Deserialize XML.
     """
-    
+
     def __init__(self, stream_or_string, **options):
         super(Deserializer, self).__init__(stream_or_string, **options)
-        self.encoding = self.options.get("encoding", settings.DEFAULT_CHARSET)
+        self.event_stream = pulldom.parse(self.stream)
-        self.event_stream = pulldom.parse(self.stream) 
+
-    
     def next(self):
         for event, node in self.event_stream:
             if event == "START_ELEMENT" and node.nodeName == "object":
                 self.event_stream.expandNode(node)
                 return self._handle_object(node)
         raise StopIteration
-        
+
     def _handle_object(self, node):
         """
         Convert an <object> node to a DeserializedObject.
         """
-        # Look up the model using the model loading mechanism. If this fails, bail.
+        # Look up the model using the model loading mechanism. If this fails,
+        # bail.
         Model = self._get_model_from_node(node, "model")
-        
+
         # Start building a data dictionary from the object.  If the node is
         # missing the pk attribute, bail.
         pk = node.getAttribute("pk")
@@ -146,11 +147,11 @@ class Deserializer(base.Deserializer):
             raise base.DeserializationError("<object> node is missing the 'pk' attribute")
 
         data = {Model._meta.pk.attname : Model._meta.pk.to_python(pk)}
-        
+
         # Also start building a dict of m2m data (this is saved as
         # {m2m_accessor_attribute : [list_of_related_objects]})
         m2m_data = {}
-        
+
         # Deseralize each field.
         for field_node in node.getElementsByTagName("field"):
             # If the field is missing the name attribute, bail (are you
@@ -158,12 +159,12 @@ class Deserializer(base.Deserializer):
             field_name = field_node.getAttribute("name")
             if not field_name:
                 raise base.DeserializationError("<field> node is missing the 'name' attribute")
-            
+
             # Get the field from the Model. This will raise a
             # FieldDoesNotExist if, well, the field doesn't exist, which will
             # be propagated correctly.
             field = Model._meta.get_field(field_name)
-            
+
             # As is usually the case, relation fields get the special treatment.
             if field.rel and isinstance(field.rel, models.ManyToManyRel):
                 m2m_data[field.name] = self._handle_m2m_field_node(field_node, field)
@@ -173,12 +174,12 @@ class Deserializer(base.Deserializer):
                 if len(field_node.childNodes) == 1 and field_node.childNodes[0].nodeName == 'None':
                     value = None
                 else:
-                    value = field.to_python(getInnerText(field_node).strip().encode(self.encoding))
+                    value = field.to_python(getInnerText(field_node).strip())
                 data[field.name] = value
-        
+
         # Return a DeserializedObject so that the m2m data has a place to live.
         return base.DeserializedObject(Model(**data), m2m_data)
-        
+
     def _handle_fk_field_node(self, node, field):
         """
         Handle a <field> node for a ForeignKey
@@ -188,16 +189,16 @@ class Deserializer(base.Deserializer):
             return None
         else:
             return field.rel.to._meta.get_field(field.rel.field_name).to_python(
-                       getInnerText(node).strip().encode(self.encoding))
+                       getInnerText(node).strip())
-        
+
     def _handle_m2m_field_node(self, node, field):
         """
-        Handle a <field> node for a ManyToManyField
+        Handle a <field> node for a ManyToManyField.
         """
         return [field.rel.to._meta.pk.to_python(
-                    c.getAttribute("pk").encode(self.encoding)) 
+                    c.getAttribute("pk"))
                     for c in node.getElementsByTagName("object")]
-    
+
     def _get_model_from_node(self, node, attr):
         """
         Helper to look up a model from a <object model=...> or a <field
@@ -217,8 +218,8 @@ class Deserializer(base.Deserializer):
                 "<%s> node has invalid model identifier: '%s'" % \
                     (node.nodeName, model_identifier))
         return Model
-        
+
-    
+
 def getInnerText(node):
     """
     Get all the inner text of a DOM node (recursively).
@@ -232,4 +233,5 @@ def getInnerText(node):
             inner_text.extend(getInnerText(child))
         else:
            pass
-    return "".join(inner_text)
+    return u"".join(inner_text)
+

django/core/urlresolvers.py

@@ -9,6 +9,7 @@ a string) and returns a tuple in this format:
 
 from django.http import Http404
 from django.core.exceptions import ImproperlyConfigured, ViewDoesNotExist
+from django.utils.encoding import iri_to_uri
 from django.utils.functional import memoize
 import re
 
@@ -37,14 +38,20 @@ def get_callable(lookup_view, can_fail=False):
     If can_fail is True, lookup_view might be a URL pattern label, so errors
     during the import fail and the string is returned.
     """
-    if not callable(lookup_view):
+    try:
-        mod_name, func_name = get_mod_func(lookup_view)
+        # Bail out early if lookup_view is not ASCII. This can't be a function.
-        try:
+        lookup_view = lookup_view.encode('ascii')
-            if func_name != '':
+
-                lookup_view = getattr(__import__(mod_name, {}, {}, ['']), func_name)
+        if not callable(lookup_view):
-        except (ImportError, AttributeError):
+            mod_name, func_name = get_mod_func(lookup_view)
-            if not can_fail:
+            try:
-                raise
+                if func_name != '':
+                    lookup_view = getattr(__import__(mod_name, {}, {}, ['']), func_name)
+            except (ImportError, AttributeError):
+                if not can_fail:
+                    raise
+    except UnicodeEncodeError:
+        pass
     return lookup_view
 get_callable = memoize(get_callable, _callable_cache)
 
@@ -265,7 +272,7 @@ class RegexURLResolver(object):
         except (ImportError, AttributeError):
             raise NoReverseMatch
         if lookup_view in self.reverse_dict:
-            return ''.join([reverse_helper(part.regex, *args, **kwargs) for part in self.reverse_dict[lookup_view]])
+            return u''.join([reverse_helper(part.regex, *args, **kwargs) for part in self.reverse_dict[lookup_view]])
         raise NoReverseMatch
 
     def reverse_helper(self, lookup_view, *args, **kwargs):
@@ -279,5 +286,5 @@ def resolve(path, urlconf=None):
 def reverse(viewname, urlconf=None, args=None, kwargs=None):
     args = args or []
     kwargs = kwargs or {}
-    return '/' + get_resolver(urlconf).reverse(viewname, *args, **kwargs)
+    return iri_to_uri(u'/' + get_resolver(urlconf).reverse(viewname, *args, **kwargs))
 

django/core/validators.py

@@ -10,8 +10,9 @@ form field is required.
 
 import urllib2
 from django.conf import settings
-from django.utils.translation import gettext, gettext_lazy, ngettext
+from django.utils.translation import ugettext as _, ugettext_lazy, ungettext
 from django.utils.functional import Promise, lazy
+from django.utils.encoding import force_unicode
 import re
 
 _datere = r'\d{4}-\d{1,2}-\d{1,2}'
@@ -32,16 +33,17 @@ phone_re = re.compile(r'^[A-PR-Y0-9]{3}-[A-PR-Y0-9]{3}-[A-PR-Y0-9]{4}$', re.IGNO
 slug_re = re.compile(r'^[-\w]+$')
 url_re = re.compile(r'^https?://\S+$')
 
-lazy_inter = lazy(lambda a,b: str(a) % b, str)
+lazy_inter = lazy(lambda a,b: force_unicode(a) % b, unicode)
 
 class ValidationError(Exception):
     def __init__(self, message):
         "ValidationError can be passed a string or a list."
         if isinstance(message, list):
-            self.messages = message
+            self.messages = [force_unicode(msg) for msg in message]
         else:
             assert isinstance(message, (basestring, Promise)), ("%s should be a string" % repr(message))
-            self.messages = [message]
+            self.messages = [force_unicode(message)]
+
     def __str__(self):
         # This is needed because, without a __str__(), printing an exception
         # instance would result in this:
@@ -53,39 +55,40 @@ class CriticalValidationError(Exception):
     def __init__(self, message):
         "ValidationError can be passed a string or a list."
         if isinstance(message, list):
-            self.messages = message
+            self.messages = [force_unicode(msg) for msg in message]
         else:
             assert isinstance(message, (basestring, Promise)), ("'%s' should be a string" % message)
-            self.messages = [message]
+            self.messages = [force_unicode(message)]
+
     def __str__(self):
         return str(self.messages)
 
 def isAlphaNumeric(field_data, all_data):
     if not alnum_re.search(field_data):
-        raise ValidationError, gettext("This value must contain only letters, numbers and underscores.")
+        raise ValidationError, _("This value must contain only letters, numbers and underscores.")
 
 def isAlphaNumericURL(field_data, all_data):
     if not alnumurl_re.search(field_data):
-        raise ValidationError, gettext("This value must contain only letters, numbers, underscores, dashes or slashes.")
+        raise ValidationError, _("This value must contain only letters, numbers, underscores, dashes or slashes.")
 
 def isSlug(field_data, all_data):
     if not slug_re.search(field_data):
-        raise ValidationError, gettext("This value must contain only letters, numbers, underscores or hyphens.")
+        raise ValidationError, _("This value must contain only letters, numbers, underscores or hyphens.")
 
 def isLowerCase(field_data, all_data):
     if field_data.lower() != field_data:
-        raise ValidationError, gettext("Uppercase letters are not allowed here.")
+        raise ValidationError, _("Uppercase letters are not allowed here.")
 
 def isUpperCase(field_data, all_data):
     if field_data.upper() != field_data:
-        raise ValidationError, gettext("Lowercase letters are not allowed here.")
+        raise ValidationError, _("Lowercase letters are not allowed here.")
 
 def isCommaSeparatedIntegerList(field_data, all_data):
     for supposed_int in field_data.split(','):
         try:
             int(supposed_int)
         except ValueError:
-            raise ValidationError, gettext("Enter only digits separated by commas.")
+            raise ValidationError, _("Enter only digits separated by commas.")
 
 def isCommaSeparatedEmailList(field_data, all_data):
     """
@@ -97,32 +100,32 @@ def isCommaSeparatedEmailList(field_data, all_data):
         try:
             isValidEmail(supposed_email.strip(), '')
         except ValidationError:
-            raise ValidationError, gettext("Enter valid e-mail addresses separated by commas.")
+            raise ValidationError, _("Enter valid e-mail addresses separated by commas.")
 
 def isValidIPAddress4(field_data, all_data):
     if not ip4_re.search(field_data):
-        raise ValidationError, gettext("Please enter a valid IP address.")
+        raise ValidationError, _("Please enter a valid IP address.")
 
 def isNotEmpty(field_data, all_data):
     if field_data.strip() == '':
-        raise ValidationError, gettext("Empty values are not allowed here.")
+        raise ValidationError, _("Empty values are not allowed here.")
 
 def isOnlyDigits(field_data, all_data):
     if not field_data.isdigit():
-        raise ValidationError, gettext("Non-numeric characters aren't allowed here.")
+        raise ValidationError, _("Non-numeric characters aren't allowed here.")
 
 def isNotOnlyDigits(field_data, all_data):
     if field_data.isdigit():
-        raise ValidationError, gettext("This value can't be comprised solely of digits.")
+        raise ValidationError, _("This value can't be comprised solely of digits.")
 
 def isInteger(field_data, all_data):
     # This differs from isOnlyDigits because this accepts the negative sign
     if not integer_re.search(field_data):
-        raise ValidationError, gettext("Enter a whole number.")
+        raise ValidationError, _("Enter a whole number.")
 
 def isOnlyLetters(field_data, all_data):
     if not field_data.isalpha():
-        raise ValidationError, gettext("Only alphabetical characters are allowed here.")
+        raise ValidationError, _("Only alphabetical characters are allowed here.")
 
 def _isValidDate(date_string):
     """
@@ -137,30 +140,30 @@ def _isValidDate(date_string):
     # This check is needed because strftime is used when saving the date
     # value to the database, and strftime requires that the year be >=1900.
     if year < 1900:
-        raise ValidationError, gettext('Year must be 1900 or later.')
+        raise ValidationError, _('Year must be 1900 or later.')
     try:
         date(year, month, day)
     except ValueError, e:
-        msg = gettext('Invalid date: %s') % gettext(str(e))
+        msg = _('Invalid date: %s') % _(str(e))
         raise ValidationError, msg    
 
 def isValidANSIDate(field_data, all_data):
     if not ansi_date_re.search(field_data):
-        raise ValidationError, gettext('Enter a valid date in YYYY-MM-DD format.')
+        raise ValidationError, _('Enter a valid date in YYYY-MM-DD format.')
     _isValidDate(field_data)
 
 def isValidANSITime(field_data, all_data):
     if not ansi_time_re.search(field_data):
-        raise ValidationError, gettext('Enter a valid time in HH:MM format.')
+        raise ValidationError, _('Enter a valid time in HH:MM format.')
 
 def isValidANSIDatetime(field_data, all_data):
     if not ansi_datetime_re.search(field_data):
-        raise ValidationError, gettext('Enter a valid date/time in YYYY-MM-DD HH:MM format.')
+        raise ValidationError, _('Enter a valid date/time in YYYY-MM-DD HH:MM format.')
     _isValidDate(field_data.split()[0])
 
 def isValidEmail(field_data, all_data):
     if not email_re.search(field_data):
-        raise ValidationError, gettext('Enter a valid e-mail address.')
+        raise ValidationError, _('Enter a valid e-mail address.')
 
 def isValidImage(field_data, all_data):
     """
@@ -172,22 +175,22 @@ def isValidImage(field_data, all_data):
     try:
         content = field_data['content']
     except TypeError:
-        raise ValidationError, gettext("No file was submitted. Check the encoding type on the form.")
+        raise ValidationError, _("No file was submitted. Check the encoding type on the form.")
     try:
         Image.open(StringIO(content))
     except IOError: # Python Imaging Library doesn't recognize it as an image
-        raise ValidationError, gettext("Upload a valid image. The file you uploaded was either not an image or a corrupted image.")
+        raise ValidationError, _("Upload a valid image. The file you uploaded was either not an image or a corrupted image.")
 
 def isValidImageURL(field_data, all_data):
     uc = URLMimeTypeCheck(('image/jpeg', 'image/gif', 'image/png'))
     try:
         uc(field_data, all_data)
     except URLMimeTypeCheck.InvalidContentType:
-        raise ValidationError, gettext("The URL %s does not point to a valid image.") % field_data
+        raise ValidationError, _("The URL %s does not point to a valid image.") % field_data
 
 def isValidPhone(field_data, all_data):
     if not phone_re.search(field_data):
-        raise ValidationError, gettext('Phone numbers must be in XXX-XXX-XXXX format. "%s" is invalid.') % field_data
+        raise ValidationError, _('Phone numbers must be in XXX-XXX-XXXX format. "%s" is invalid.') % field_data
 
 def isValidQuicktimeVideoURL(field_data, all_data):
     "Checks that the given URL is a video that can be played by QuickTime (qt, mpeg)"
@@ -195,11 +198,11 @@ def isValidQuicktimeVideoURL(field_data, all_data):
     try:
         uc(field_data, all_data)
     except URLMimeTypeCheck.InvalidContentType:
-        raise ValidationError, gettext("The URL %s does not point to a valid QuickTime video.") % field_data
+        raise ValidationError, _("The URL %s does not point to a valid QuickTime video.") % field_data
 
 def isValidURL(field_data, all_data):
     if not url_re.search(field_data):
-        raise ValidationError, gettext("A valid URL is required.")
+        raise ValidationError, _("A valid URL is required.")
 
 def isValidHTML(field_data, all_data):
     import urllib, urllib2
@@ -213,14 +216,14 @@ def isValidHTML(field_data, all_data):
         return
     from xml.dom.minidom import parseString
     error_messages = [e.firstChild.wholeText for e in parseString(u.read()).getElementsByTagName('messages')[0].getElementsByTagName('msg')]
-    raise ValidationError, gettext("Valid HTML is required. Specific errors are:\n%s") % "\n".join(error_messages)
+    raise ValidationError, _("Valid HTML is required. Specific errors are:\n%s") % "\n".join(error_messages)
 
 def isWellFormedXml(field_data, all_data):
     from xml.dom.minidom import parseString
     try:
         parseString(field_data)
     except Exception, e: # Naked except because we're not sure what will be thrown
-        raise ValidationError, gettext("Badly formed XML: %s") % str(e)
+        raise ValidationError, _("Badly formed XML: %s") % str(e)
 
 def isWellFormedXmlFragment(field_data, all_data):
     isWellFormedXml('<root>%s</root>' % field_data, all_data)
@@ -250,7 +253,7 @@ def isValidUSState(field_data, all_data):
     "Checks that the given string is a valid two-letter U.S. state abbreviation"
     states = ['AA', 'AE', 'AK', 'AL', 'AP', 'AR', 'AS', 'AZ', 'CA', 'CO', 'CT', 'DC', 'DE', 'FL', 'FM', 'GA', 'GU', 'HI', 'IA', 'ID', 'IL', 'IN', 'KS', 'KY', 'LA', 'MA', 'MD', 'ME', 'MH', 'MI', 'MN', 'MO', 'MP', 'MS', 'MT', 'NC', 'ND', 'NE', 'NH', 'NJ', 'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'PR', 'PW', 'RI', 'SC', 'SD', 'TN', 'TX', 'UT', 'VA', 'VI', 'VT', 'WA', 'WI', 'WV', 'WY']
     if field_data.upper() not in states:
-        raise ValidationError, gettext("Enter a valid U.S. state abbreviation.")
+        raise ValidationError, _("Enter a valid U.S. state abbreviation.")
 
 def hasNoProfanities(field_data, all_data):
     """
@@ -264,14 +267,14 @@ def hasNoProfanities(field_data, all_data):
     if words_seen:
         from django.utils.text import get_text_list
         plural = len(words_seen) > 1
-        raise ValidationError, ngettext("Watch your mouth! The word %s is not allowed here.",
+        raise ValidationError, ungettext("Watch your mouth! The word %s is not allowed here.",
             "Watch your mouth! The words %s are not allowed here.", plural) % \
             get_text_list(['"%s%s%s"' % (i[0], '-'*(len(i)-2), i[-1]) for i in words_seen], 'and')
 
 class AlwaysMatchesOtherField(object):
     def __init__(self, other_field_name, error_message=None):
         self.other = other_field_name
-        self.error_message = error_message or lazy_inter(gettext_lazy("This field must match the '%s' field."), self.other)
+        self.error_message = error_message or lazy_inter(ugettext_lazy("This field must match the '%s' field."), self.other)
         self.always_test = True
 
     def __call__(self, field_data, all_data):
@@ -290,7 +293,7 @@ class ValidateIfOtherFieldEquals(object):
                 v(field_data, all_data)
 
 class RequiredIfOtherFieldNotGiven(object):
-    def __init__(self, other_field_name, error_message=gettext_lazy("Please enter something for at least one field.")):
+    def __init__(self, other_field_name, error_message=ugettext_lazy("Please enter something for at least one field.")):
         self.other, self.error_message = other_field_name, error_message
         self.always_test = True
 
@@ -299,7 +302,7 @@ class RequiredIfOtherFieldNotGiven(object):
             raise ValidationError, self.error_message
 
 class RequiredIfOtherFieldsGiven(object):
-    def __init__(self, other_field_names, error_message=gettext_lazy("Please enter both fields or leave them both empty.")):
+    def __init__(self, other_field_names, error_message=ugettext_lazy("Please enter both fields or leave them both empty.")):
         self.other, self.error_message = other_field_names, error_message
         self.always_test = True
 
@@ -310,7 +313,7 @@ class RequiredIfOtherFieldsGiven(object):
 
 class RequiredIfOtherFieldGiven(RequiredIfOtherFieldsGiven):
     "Like RequiredIfOtherFieldsGiven, but takes a single field name instead of a list."
-    def __init__(self, other_field_name, error_message=gettext_lazy("Please enter both fields or leave them both empty.")):
+    def __init__(self, other_field_name, error_message=ugettext_lazy("Please enter both fields or leave them both empty.")):
         RequiredIfOtherFieldsGiven.__init__(self, [other_field_name], error_message)
 
 class RequiredIfOtherFieldEquals(object):
@@ -318,7 +321,7 @@ class RequiredIfOtherFieldEquals(object):
         self.other_field = other_field
         self.other_value = other_value
         other_label = other_label or other_value
-        self.error_message = error_message or lazy_inter(gettext_lazy("This field must be given if %(field)s is %(value)s"), {
+        self.error_message = error_message or lazy_inter(ugettext_lazy("This field must be given if %(field)s is %(value)s"), {
             'field': other_field, 'value': other_label})
         self.always_test = True
 
@@ -331,7 +334,7 @@ class RequiredIfOtherFieldDoesNotEqual(object):
         self.other_field = other_field
         self.other_value = other_value
         other_label = other_label or other_value
-        self.error_message = error_message or lazy_inter(gettext_lazy("This field must be given if %(field)s is not %(value)s"), {
+        self.error_message = error_message or lazy_inter(ugettext_lazy("This field must be given if %(field)s is not %(value)s"), {
             'field': other_field, 'value': other_label})
         self.always_test = True
 
@@ -350,7 +353,7 @@ class IsLessThanOtherField(object):
 class UniqueAmongstFieldsWithPrefix(object):
     def __init__(self, field_name, prefix, error_message):
         self.field_name, self.prefix = field_name, prefix
-        self.error_message = error_message or gettext_lazy("Duplicate values are not allowed.")
+        self.error_message = error_message or ugettext_lazy("Duplicate values are not allowed.")
 
     def __call__(self, field_data, all_data):
         for field_name, value in all_data.items():
@@ -365,11 +368,11 @@ class NumberIsInRange(object):
         self.lower, self.upper = lower, upper
         if not error_message:
             if lower and upper:
-                 self.error_message = gettext("This value must be between %(lower)s and %(upper)s.") % {'lower': lower, 'upper': upper}
+                 self.error_message = _("This value must be between %(lower)s and %(upper)s.") % {'lower': lower, 'upper': upper}
             elif lower:
-                self.error_message = gettext("This value must be at least %s.") % lower
+                self.error_message = _("This value must be at least %s.") % lower
             elif upper:
-                self.error_message = gettext("This value must be no more than %s.") % upper
+                self.error_message = _("This value must be no more than %s.") % upper
         else:
             self.error_message = error_message
 
@@ -405,7 +408,7 @@ class IsAPowerOf(object):
         from math import log
         val = log(int(field_data)) / log(self.power_of)
         if val != int(val):
-            raise ValidationError, gettext("This value must be a power of %s.") % self.power_of
+            raise ValidationError, _("This value must be a power of %s.") % self.power_of
 
 class IsValidDecimal(object):
     def __init__(self, max_digits, decimal_places):
@@ -414,19 +417,19 @@ class IsValidDecimal(object):
     def __call__(self, field_data, all_data):
         match = decimal_re.search(str(field_data))
         if not match:
-            raise ValidationError, gettext("Please enter a valid decimal number.")
+            raise ValidationError, _("Please enter a valid decimal number.")
         
         digits = len(match.group('digits') or '')
         decimals = len(match.group('decimals') or '')
         
         if digits + decimals > self.max_digits:
-            raise ValidationError, ngettext("Please enter a valid decimal number with at most %s total digit.",
+            raise ValidationError, ungettext("Please enter a valid decimal number with at most %s total digit.",
                 "Please enter a valid decimal number with at most %s total digits.", self.max_digits) % self.max_digits
         if digits > (self.max_digits - self.decimal_places):
-            raise ValidationError, ngettext( "Please enter a valid decimal number with a whole part of at most %s digit.",
+            raise ValidationError, ungettext( "Please enter a valid decimal number with a whole part of at most %s digit.",
                 "Please enter a valid decimal number with a whole part of at most %s digits.", str(self.max_digits-self.decimal_places)) % str(self.max_digits-self.decimal_places)
         if decimals > self.decimal_places:
-            raise ValidationError, ngettext("Please enter a valid decimal number with at most %s decimal place.",
+            raise ValidationError, ungettext("Please enter a valid decimal number with at most %s decimal place.",
                 "Please enter a valid decimal number with at most %s decimal places.", self.decimal_places) % self.decimal_places
 
 def isValidFloat(field_data, all_data):
@@ -434,7 +437,7 @@ def isValidFloat(field_data, all_data):
     try:
         float(data)
     except ValueError:
-        raise ValidationError, gettext("Please enter a valid floating point number.")
+        raise ValidationError, ugettext("Please enter a valid floating point number.")
 
 class HasAllowableSize(object):
     """
@@ -443,14 +446,14 @@ class HasAllowableSize(object):
     """
     def __init__(self, min_size=None, max_size=None, min_error_message=None, max_error_message=None):
         self.min_size, self.max_size = min_size, max_size
-        self.min_error_message = min_error_message or lazy_inter(gettext_lazy("Make sure your uploaded file is at least %s bytes big."), min_size)
+        self.min_error_message = min_error_message or lazy_inter(ugettext_lazy("Make sure your uploaded file is at least %s bytes big."), min_size)
-        self.max_error_message = max_error_message or lazy_inter(gettext_lazy("Make sure your uploaded file is at most %s bytes big."), max_size)
+        self.max_error_message = max_error_message or lazy_inter(ugettext_lazy("Make sure your uploaded file is at most %s bytes big."), max_size)
 
     def __call__(self, field_data, all_data):
         try:
             content = field_data['content']
         except TypeError:
-            raise ValidationError, gettext_lazy("No file was submitted. Check the encoding type on the form.")
+            raise ValidationError, ugettext_lazy("No file was submitted. Check the encoding type on the form.")
         if self.min_size is not None and len(content) < self.min_size:
             raise ValidationError, self.min_error_message
         if self.max_size is not None and len(content) > self.max_size:
@@ -461,7 +464,7 @@ class MatchesRegularExpression(object):
     Checks that the field matches the given regular-expression. The regex
     should be in string format, not already compiled.
     """
-    def __init__(self, regexp, error_message=gettext_lazy("The format for this field is wrong.")):
+    def __init__(self, regexp, error_message=ugettext_lazy("The format for this field is wrong.")):
         self.regexp = re.compile(regexp)
         self.error_message = error_message
 
@@ -476,7 +479,7 @@ class AnyValidator(object):
     as a validation error. The message is rather unspecific, so it's best to
     specify one on instantiation.
     """
-    def __init__(self, validator_list=None, error_message=gettext_lazy("This field is invalid.")):
+    def __init__(self, validator_list=None, error_message=ugettext_lazy("This field is invalid.")):
         if validator_list is None: validator_list = []
         self.validator_list = validator_list
         self.error_message = error_message
@@ -512,10 +515,10 @@ class URLMimeTypeCheck(object):
         try:
             info = urllib2.urlopen(field_data).info()
         except (urllib2.HTTPError, urllib2.URLError):
-            raise URLMimeTypeCheck.CouldNotRetrieve, gettext("Could not retrieve anything from %s.") % field_data
+            raise URLMimeTypeCheck.CouldNotRetrieve, _("Could not retrieve anything from %s.") % field_data
         content_type = info['content-type']
         if content_type not in self.mime_type_list:
-            raise URLMimeTypeCheck.InvalidContentType, gettext("The URL %(url)s returned the invalid Content-Type header '%(contenttype)s'.") % {
+            raise URLMimeTypeCheck.InvalidContentType, _("The URL %(url)s returned the invalid Content-Type header '%(contenttype)s'.") % {
                 'url': field_data, 'contenttype': content_type}
 
 class RelaxNGCompact(object):

django/db/backends/mysql/base.py

@@ -84,7 +84,7 @@ class DatabaseWrapper(local):
             kwargs = {
                 'conv': django_conversions,
                 'charset': 'utf8',
-                'use_unicode': False,
+                'use_unicode': True,
             }
             if settings.DATABASE_USER:
                 kwargs['user'] = settings.DATABASE_USER

django/db/backends/mysql_old/base.py

@@ -91,6 +91,7 @@ class DatabaseWrapper(local):
                 'db': settings.DATABASE_NAME,
                 'passwd': settings.DATABASE_PASSWORD,
                 'conv': django_conversions,
+                'use_unicode': True,
             }
             if settings.DATABASE_HOST.startswith('/'):
                 kwargs['unix_socket'] = settings.DATABASE_HOST
@@ -103,6 +104,7 @@ class DatabaseWrapper(local):
             cursor = self.connection.cursor()
             if self.connection.get_server_info() >= '4.1':
                 cursor.execute("SET NAMES 'utf8'")
+                cursor.execute("SET CHARACTER SET 'utf8'")
         else:
             cursor = self.connection.cursor()
         if settings.DEBUG:

django/db/backends/oracle/base.py

@@ -6,13 +6,18 @@ Requires cx_Oracle: http://www.python.net/crew/atuining/cx_Oracle/
 
 from django.conf import settings
 from django.db.backends import util
+from django.utils.datastructures import SortedDict
+from django.utils.encoding import smart_str, force_unicode
+import datetime
+import os
+
+# Oracle takes client-side character set encoding from the environment.
+os.environ['NLS_LANG'] = '.UTF8'
 try:
     import cx_Oracle as Database
 except ImportError, e:
     from django.core.exceptions import ImproperlyConfigured
     raise ImproperlyConfigured, "Error loading cx_Oracle module: %s" % e
-import datetime
-from django.utils.datastructures import SortedDict
 
 
 DatabaseError = Database.Error
@@ -45,9 +50,9 @@ class DatabaseWrapper(local):
                 conn_string = "%s/%s@%s" % (settings.DATABASE_USER, settings.DATABASE_PASSWORD, settings.DATABASE_NAME)
                 self.connection = Database.connect(conn_string, **self.options)
         cursor = FormatStylePlaceholderCursor(self.connection)
-        # default arraysize of 1 is highly sub-optimal
+        # Default arraysize of 1 is highly sub-optimal.
         cursor.arraysize = 100
-        # set oracle date to ansi date format
+        # Set oracle date to ansi date format.
         cursor.execute("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD'")
         cursor.execute("ALTER SESSION SET NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS.FF'")
         if settings.DEBUG:
@@ -78,23 +83,22 @@ uses_case_insensitive_names = True
 
 class FormatStylePlaceholderCursor(Database.Cursor):
     """
-    Django uses "format" (e.g. '%s') style placeholders, but Oracle uses ":var" style.
+    Django uses "format" (e.g. '%s') style placeholders, but Oracle uses ":var"
-    This fixes it -- but note that if you want to use a literal "%s" in a query,
+    style. This fixes it -- but note that if you want to use a literal "%s" in
-    you'll need to use "%%s".
+    a query, you'll need to use "%%s".
+
+    We also do automatic conversion between Unicode on the Python side and
+    UTF-8 -- for talking to Oracle -- in here.
     """
+    charset = 'utf-8'
+
     def _rewrite_args(self, query, params=None):
         if params is None:
             params = []
         else:
-            # cx_Oracle can't handle unicode parameters, so cast to str for now
+            params = self._format_params(params)
-            for i, param in enumerate(params):
-                if type(param) == unicode:
-                    try:
-                        params[i] = param.encode('utf-8')
-                    except UnicodeError:
-                        params[i] = str(param)
         args = [(':arg%d' % i) for i in range(len(params))]
-        query = query % tuple(args)
+        query = smart_str(query, self.charset) % tuple(args)
         # cx_Oracle wants no trailing ';' for SQL statements.  For PL/SQL, it
         # it does want a trailing ';' but not a trailing '/'.  However, these
         # characters must be included in the original query in case the query
@@ -103,6 +107,16 @@ class FormatStylePlaceholderCursor(Database.Cursor):
             query = query[:-1]
         return query, params
 
+    def _format_params(self, params):
+        if isinstance(params, dict):
+            result = {}
+            charset = self.charset
+            for key, value in params.items():
+                result[smart_str(key, charset)] = smart_str(value, charset)
+            return result
+        else:
+            return tuple([smart_str(p, self.charset, True) for p in params])
+
     def execute(self, query, params=None):
         query, params = self._rewrite_args(query, params)
         return Database.Cursor.execute(self, query, params)
@@ -111,6 +125,26 @@ class FormatStylePlaceholderCursor(Database.Cursor):
         query, params = self._rewrite_args(query, params)
         return Database.Cursor.executemany(self, query, params)
 
+    def fetchone(self):
+        return to_unicode(Database.Cursor.fetchone(self))
+
+    def fetchmany(self, size=None):
+        if size is None:
+            size = self.arraysize
+        return tuple([tuple([to_unicode(e) for e in r]) for r in Database.Cursor.fetchmany(self, size)])
+
+    def fetchall(self):
+        return tuple([tuple([to_unicode(e) for e in r]) for r in Database.Cursor.fetchall(self)])
+
+def to_unicode(s):
+    """
+    Convert strings to Unicode objects (and return all other data types
+    unchanged).
+    """
+    if isinstance(s, basestring):
+        return force_unicode(s)
+    return s
+
 def quote_name(name):
     # SQL92 requires delimited (quoted) names to be case-sensitive.  When
     # not quoted, Oracle has case-insensitive behavior for identifiers, but

django/db/backends/oracle/creation.py

@@ -8,15 +8,15 @@ from django.core import management
 DATA_TYPES = {
     'AutoField':                    'NUMBER(11)',
     'BooleanField':                 'NUMBER(1) CHECK (%(column)s IN (0,1))',
-    'CharField':                    'VARCHAR2(%(maxlength)s)',
+    'CharField':                    'NVARCHAR2(%(maxlength)s)',
     'CommaSeparatedIntegerField':   'VARCHAR2(%(maxlength)s)',
     'DateField':                    'DATE',
     'DateTimeField':                'TIMESTAMP',
     'DecimalField':                 'NUMBER(%(max_digits)s, %(decimal_places)s)',
-    'FileField':                    'VARCHAR2(100)',
+    'FileField':                    'NVARCHAR2(100)',
-    'FilePathField':                'VARCHAR2(100)',
+    'FilePathField':                'NVARCHAR2(100)',
     'FloatField':                   'DOUBLE PRECISION',
-    'ImageField':                   'VARCHAR2(100)',
+    'ImageField':                   'NVARCHAR2(100)',
     'IntegerField':                 'NUMBER(11)',
     'IPAddressField':               'VARCHAR2(15)',
     'ManyToManyField':              None,
@@ -25,7 +25,7 @@ DATA_TYPES = {
     'PhoneNumberField':             'VARCHAR2(20)',
     'PositiveIntegerField':         'NUMBER(11) CHECK (%(column)s >= 0)',
     'PositiveSmallIntegerField':    'NUMBER(11) CHECK (%(column)s >= 0)',
-    'SlugField':                    'VARCHAR2(50)',
+    'SlugField':                    'NVARCHAR2(50)',
     'SmallIntegerField':            'NUMBER(11)',
     'TextField':                    'NCLOB',
     'TimeField':                    'TIMESTAMP',

django/db/backends/postgresql/base.py

@@ -4,6 +4,7 @@ PostgreSQL database backend for Django.
 Requires psycopg 1: http://initd.org/projects/psycopg1
 """
 
+from django.utils.encoding import smart_str, smart_unicode
 from django.db.backends import util
 try:
     import psycopg as Database
@@ -21,11 +22,6 @@ except ImportError:
     # Import copy of _thread_local.py from Python 2.4
     from django.utils._threading_local import local
 
-def smart_basestring(s, charset):
-    if isinstance(s, unicode):
-        return s.encode(charset)
-    return s
-
 class UnicodeCursorWrapper(object):
     """
     A thin wrapper around psycopg cursors that allows them to accept Unicode
@@ -33,18 +29,31 @@ class UnicodeCursorWrapper(object):
 
     This is necessary because psycopg doesn't apply any DB quoting to
     parameters that are Unicode strings. If a param is Unicode, this will
-    convert it to a bytestring using DEFAULT_CHARSET before passing it to
+    convert it to a bytestring using database client's encoding before passing
-    psycopg.
+    it to psycopg.
+
+    All results retrieved from the database are converted into Unicode strings
+    before being returned to the caller.
     """
     def __init__(self, cursor, charset):
         self.cursor = cursor
         self.charset = charset
 
+    def format_params(self, params):
+        if isinstance(params, dict):
+            result = {}
+            charset = self.charset
+            for key, value in params.items():
+                result[smart_str(key, charset)] = smart_str(value, charset)
+            return result
+        else:
+            return tuple([smart_str(p, self.charset, True) for p in params])
+
     def execute(self, sql, params=()):
-        return self.cursor.execute(sql, [smart_basestring(p, self.charset) for p in params])
+        return self.cursor.execute(smart_str(sql, self.charset), self.format_params(params))
 
     def executemany(self, sql, param_list):
-        new_param_list = [tuple([smart_basestring(p, self.charset) for p in params]) for params in param_list]
+        new_param_list = [self.format_params(params) for params in param_list]
         return self.cursor.executemany(sql, new_param_list)
 
     def __getattr__(self, attr):
@@ -83,7 +92,8 @@ class DatabaseWrapper(local):
         cursor = self.connection.cursor()
         if set_tz:
             cursor.execute("SET TIME ZONE %s", [settings.TIME_ZONE])
-        cursor = UnicodeCursorWrapper(cursor, settings.DEFAULT_CHARSET)
+        cursor.execute("SET client_encoding to 'UNICODE'")
+        cursor = UnicodeCursorWrapper(cursor, 'utf-8')
         global postgres_version
         if not postgres_version:
             cursor.execute("SELECT version()")
@@ -186,16 +196,17 @@ def get_sql_flush(style, tables, sequences):
     """
     if tables:
         if postgres_version[0] >= 8 and postgres_version[1] >= 1:
-            # Postgres 8.1+ can do 'TRUNCATE x, y, z...;'. In fact, it *has to* in order to be able to
+            # Postgres 8.1+ can do 'TRUNCATE x, y, z...;'. In fact, it *has to*
-            # truncate tables referenced by a foreign key in any other table. The result is a
+            # in order to be able to truncate tables referenced by a foreign
-            # single SQL TRUNCATE statement.
+            # key in any other table. The result is a single SQL TRUNCATE
+            # statement.
             sql = ['%s %s;' % \
                 (style.SQL_KEYWORD('TRUNCATE'),
                  style.SQL_FIELD(', '.join([quote_name(table) for table in tables]))
             )]
         else:
-            # Older versions of Postgres can't do TRUNCATE in a single call, so they must use
+            # Older versions of Postgres can't do TRUNCATE in a single call, so
-            # a simple delete.
+            # they must use a simple delete.
             sql = ['%s %s %s;' % \
                     (style.SQL_KEYWORD('DELETE'),
                      style.SQL_KEYWORD('FROM'),
@@ -263,6 +274,14 @@ def get_sql_sequence_reset(style, model_list):
                 style.SQL_TABLE(f.m2m_db_table())))
     return output
 
+def typecast_string(s):
+    """
+    Cast all returned strings to unicode strings.
+    """
+    if not s:
+        return s
+    return smart_unicode(s)
+
 # Register these custom typecasts, because Django expects dates/times to be
 # in Python's native (standard-library) datetime/time format, whereas psycopg
 # use mx.DateTime by default.
@@ -274,6 +293,7 @@ Database.register_type(Database.new_type((1083,1266), "TIME", util.typecast_time
 Database.register_type(Database.new_type((1114,1184), "TIMESTAMP", util.typecast_timestamp))
 Database.register_type(Database.new_type((16,), "BOOLEAN", util.typecast_boolean))
 Database.register_type(Database.new_type((1700,), "NUMERIC", util.typecast_decimal))
+Database.register_type(Database.new_type(Database.types[1043].values, 'STRING', typecast_string))
 
 OPERATOR_MAPPING = {
     'exact': '= %s',

django/db/backends/postgresql_psycopg2/base.py

@@ -7,6 +7,7 @@ Requires psycopg 2: http://initd.org/projects/psycopg2
 from django.db.backends import util
 try:
     import psycopg2 as Database
+    import psycopg2.extensions
 except ImportError, e:
     from django.core.exceptions import ImproperlyConfigured
     raise ImproperlyConfigured, "Error loading psycopg2 module: %s" % e
@@ -21,6 +22,8 @@ except ImportError:
     # Import copy of _thread_local.py from Python 2.4
     from django.utils._threading_local import local
 
+psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
+
 postgres_version = None
 
 class DatabaseWrapper(local):
@@ -48,6 +51,7 @@ class DatabaseWrapper(local):
                 conn_string += " port=%s" % settings.DATABASE_PORT
             self.connection = Database.connect(conn_string, **self.options)
             self.connection.set_isolation_level(1) # make transactions transparent to all cursors
+            self.connection.set_client_encoding('UTF8')
         cursor = self.connection.cursor()
         cursor.tzinfo_factory = None
         if set_tz:

django/db/backends/sqlite3/base.py

@@ -34,14 +34,6 @@ Database.register_converter("TIMESTAMP", util.typecast_timestamp)
 Database.register_converter("decimal", util.typecast_decimal)
 Database.register_adapter(decimal.Decimal, util.rev_typecast_decimal)
 
-def utf8rowFactory(cursor, row):
-    def utf8(s):
-        if type(s) == unicode:
-            return s.encode("utf-8")
-        else:
-            return s
-    return [utf8(r) for r in row]
-
 try:
     # Only exists in Python 2.4+
     from threading import local
@@ -69,7 +61,6 @@ class DatabaseWrapper(local):
             self.connection.create_function("django_date_trunc", 2, _sqlite_date_trunc)
             self.connection.create_function("regexp", 2, _sqlite_regexp)
         cursor = self.connection.cursor(factory=SQLiteCursorWrapper)
-        cursor.row_factory = utf8rowFactory
         if settings.DEBUG:
             return util.CursorDebugWrapper(cursor, self)
         else:
@@ -85,8 +76,9 @@ class DatabaseWrapper(local):
 
     def close(self):
         from django.conf import settings
-        # If database is in memory, closing the connection destroys the database.
+        # If database is in memory, closing the connection destroys the
-        # To prevent accidental data loss, ignore close requests on an in-memory db.
+        # database.  To prevent accidental data loss, ignore close requests on
+        # an in-memory db.
         if self.connection is not None and settings.DATABASE_NAME != ":memory:":
             self.connection.close()
             self.connection = None
@@ -181,10 +173,10 @@ def get_autoinc_sql(table):
     return None
 
 def get_sql_flush(style, tables, sequences):
-    """Return a list of SQL statements required to remove all data from
+    """
+    Return a list of SQL statements required to remove all data from
     all tables in the database (without actually removing the tables
     themselves) and put the database in an empty 'initial' state
-
     """
     # NB: The generated SQL below is specific to SQLite
     # Note: The DELETE FROM... SQL generated below works for SQLite databases
@@ -241,3 +233,4 @@ OPERATOR_MAPPING = {
     'istartswith': "LIKE %s ESCAPE '\\'",
     'iendswith': "LIKE %s ESCAPE '\\'",
 }
+

django/db/backends/util.py

@@ -1,6 +1,7 @@
 import datetime
 import md5
 from time import time
+from django.utils.encoding import smart_unicode, force_unicode
 
 try:
     import decimal
@@ -18,12 +19,8 @@ class CursorDebugWrapper(object):
             return self.cursor.execute(sql, params)
         finally:
             stop = time()
-            # If params was a list, convert it to a tuple, because string
-            # formatting with '%' only works with tuples or dicts.
-            if not isinstance(params, (tuple, dict)):
-                params = tuple(params)
             self.db.queries.append({
-                'sql': sql % params,
+                'sql': smart_unicode(sql) % convert_args(params),
                 'time': "%.3f" % (stop - start),
             })
 
@@ -34,7 +31,7 @@ class CursorDebugWrapper(object):
         finally:
             stop = time()
             self.db.queries.append({
-                'sql': 'MANY: ' + sql + ' ' + str(tuple(param_list)),
+                'sql': 'MANY: ' + sql + ' ' + smart_unicode(tuple(param_list)),
                 'time': "%.3f" % (stop - start),
             })
 
@@ -44,6 +41,16 @@ class CursorDebugWrapper(object):
         else:
             return getattr(self.cursor, attr)
 
+def convert_args(args):
+    """
+    Convert sequence or dictionary to contain unicode values.
+    """
+    to_unicode = lambda s: force_unicode(s, strings_only=True)
+    if isinstance(args, (list, tuple)):
+        return tuple([to_unicode(val) for val in args])
+    else:
+        return dict([(to_unicode(k), to_unicode(v)) for k, v in args.items()])
+
 ###############################################
 # Converters from database (string) to Python #
 ###############################################

django/db/models/base.py

@@ -12,6 +12,7 @@ from django.db.models.loading import register_models, get_model
 from django.dispatch import dispatcher
 from django.utils.datastructures import SortedDict
 from django.utils.functional import curry
+from django.utils.encoding import smart_str, force_unicode
 from django.conf import settings
 from itertools import izip
 import types
@@ -83,9 +84,11 @@ class Model(object):
         return getattr(self, self._meta.pk.attname)
 
     def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, self)
+        return smart_str(u'<%s: %s>' % (self.__class__.__name__, unicode(self)))
 
     def __str__(self):
+        if hasattr(self, '__unicode__'):
+            return force_unicode(self).encode('utf-8')
         return '%s object' % self.__class__.__name__
 
     def __eq__(self, other):
@@ -318,14 +321,14 @@ class Model(object):
 
     def _get_FIELD_display(self, field):
         value = getattr(self, field.attname)
-        return dict(field.choices).get(value, value)
+        return force_unicode(dict(field.choices).get(value, value), strings_only=True)
 
     def _get_next_or_previous_by_FIELD(self, field, is_next, **kwargs):
         op = is_next and '>' or '<'
         where = '(%s %s %%s OR (%s = %%s AND %s.%s %s %%s))' % \
             (backend.quote_name(field.column), op, backend.quote_name(field.column),
             backend.quote_name(self._meta.db_table), backend.quote_name(self._meta.pk.column), op)
-        param = str(getattr(self, field.attname))
+        param = smart_str(getattr(self, field.attname))
         q = self.__class__._default_manager.filter(**kwargs).order_by((not is_next and '-' or '') + field.name, (not is_next and '-' or '') + self._meta.pk.name)
         q._where.append(where)
         q._params.extend([param, param, getattr(self, self._meta.pk.attname)])

django/db/models/fields/__init__.py

@@ -8,7 +8,8 @@ from django.core.exceptions import ObjectDoesNotExist
 from django.utils.functional import curry
 from django.utils.itercompat import tee
 from django.utils.text import capfirst
-from django.utils.translation import gettext, gettext_lazy
+from django.utils.translation import ugettext_lazy, ugettext as _
+from django.utils.encoding import smart_unicode
 import datetime, os, time
 try:
     import decimal
@@ -26,7 +27,7 @@ BLANK_CHOICE_DASH = [("", "---------")]
 BLANK_CHOICE_NONE = [("", "None")]
 
 # prepares a value for use in a LIKE query
-prep_for_like_query = lambda x: str(x).replace("\\", "\\\\").replace("%", "\%").replace("_", "\_")
+prep_for_like_query = lambda x: smart_unicode(x).replace("\\", "\\\\").replace("%", "\%").replace("_", "\_")
 
 # returns the <ul> class for a given radio_admin value
 get_ul_class = lambda x: 'radiolist%s' % ((x == HORIZONTAL) and ' inline' or '')
@@ -43,7 +44,7 @@ def manipulator_validator_unique(f, opts, self, field_data, all_data):
         return
     if getattr(self, 'original_object', None) and self.original_object._get_pk_val() == old_obj._get_pk_val():
         return
-    raise validators.ValidationError, gettext("%(optname)s with this %(fieldname)s already exists.") % {'optname': capfirst(opts.verbose_name), 'fieldname': f.verbose_name}
+    raise validators.ValidationError, _("%(optname)s with this %(fieldname)s already exists.") % {'optname': capfirst(opts.verbose_name), 'fieldname': f.verbose_name}
 
 # A guide to Field parameters:
 #
@@ -123,7 +124,7 @@ class Field(object):
         Subclasses should implement validate(), not validate_full().
         """
         if not self.blank and not field_data:
-            return [gettext_lazy('This field is required.')]
+            return [_('This field is required.')]
         try:
             self.validate(field_data, all_data)
         except validators.ValidationError, e:
@@ -280,7 +281,7 @@ class Field(object):
                     core_field_names.extend(f.get_manipulator_field_names(name_prefix))
             # Now, if there are any, add the validator to this FormField.
             if core_field_names:
-                params['validator_list'].append(validators.RequiredIfOtherFieldsGiven(core_field_names, gettext_lazy("This field is required.")))
+                params['validator_list'].append(validators.RequiredIfOtherFieldsGiven(core_field_names, ugettext_lazy("This field is required.")))
 
         # Finally, add the field_names.
         field_names = self.get_manipulator_field_names(name_prefix)
@@ -308,9 +309,9 @@ class Field(object):
             return first_choice + list(self.choices)
         rel_model = self.rel.to
         if hasattr(self.rel, 'get_related_field'):
-            lst = [(getattr(x, self.rel.get_related_field().attname), str(x)) for x in rel_model._default_manager.complex_filter(self.rel.limit_choices_to)]
+            lst = [(getattr(x, self.rel.get_related_field().attname), smart_unicode(x)) for x in rel_model._default_manager.complex_filter(self.rel.limit_choices_to)]
         else:
-            lst = [(x._get_pk_val(), str(x)) for x in rel_model._default_manager.complex_filter(self.rel.limit_choices_to)]
+            lst = [(x._get_pk_val(), smart_unicode(x)) for x in rel_model._default_manager.complex_filter(self.rel.limit_choices_to)]
         return first_choice + lst
 
     def get_choices_default(self):
@@ -375,7 +376,7 @@ class AutoField(Field):
         try:
             return int(value)
         except (TypeError, ValueError):
-            raise validators.ValidationError, gettext("This value must be an integer.")
+            raise validators.ValidationError, _("This value must be an integer.")
 
     def get_manipulator_fields(self, opts, manipulator, change, name_prefix='', rel=False, follow=True):
         if not rel:
@@ -410,7 +411,7 @@ class BooleanField(Field):
         if value in (True, False): return value
         if value in ('t', 'True', '1'): return True
         if value in ('f', 'False', '0'): return False
-        raise validators.ValidationError, gettext("This value must be either True or False.")
+        raise validators.ValidationError, _("This value must be either True or False.")
 
     def get_manipulator_field_objs(self):
         return [oldforms.CheckboxField]
@@ -431,8 +432,8 @@ class CharField(Field):
             if self.null:
                 return value
             else:
-                raise validators.ValidationError, gettext_lazy("This field cannot be null.")
+                raise validators.ValidationError, ugettext_lazy("This field cannot be null.")
-        return str(value)
+        return smart_unicode(value)
 
     def formfield(self, **kwargs):
         defaults = {'max_length': self.maxlength}
@@ -465,15 +466,15 @@ class DateField(Field):
         try:
             return datetime.date(*time.strptime(value, '%Y-%m-%d')[:3])
         except ValueError:
-            raise validators.ValidationError, gettext('Enter a valid date in YYYY-MM-DD format.')
+            raise validators.ValidationError, _('Enter a valid date in YYYY-MM-DD format.')
 
     def get_db_prep_lookup(self, lookup_type, value):
         if lookup_type == 'range':
-            value = [str(v) for v in value]
+            value = [smart_unicode(v) for v in value]
         elif lookup_type in ('exact', 'gt', 'gte', 'lt', 'lte') and hasattr(value, 'strftime'):
             value = value.strftime('%Y-%m-%d')
         else:
-            value = str(value)
+            value = smart_unicode(value)
         return Field.get_db_prep_lookup(self, lookup_type, value)
 
     def pre_save(self, model_instance, add):
@@ -534,7 +535,7 @@ class DateTimeField(DateField):
                 try:
                     return datetime.datetime(*time.strptime(value, '%Y-%m-%d')[:3])
                 except ValueError:
-                    raise validators.ValidationError, gettext('Enter a valid date/time in YYYY-MM-DD HH:MM format.')
+                    raise validators.ValidationError, _('Enter a valid date/time in YYYY-MM-DD HH:MM format.')
 
     def get_db_prep_save(self, value):
         # Casts dates into string format for entry into database.
@@ -543,14 +544,14 @@ class DateTimeField(DateField):
             # doesn't support microseconds.
             if settings.DATABASE_ENGINE == 'mysql' and hasattr(value, 'microsecond'):
                 value = value.replace(microsecond=0)
-            value = str(value)
+            value = smart_unicode(value)
         return Field.get_db_prep_save(self, value)
 
     def get_db_prep_lookup(self, lookup_type, value):
         if lookup_type == 'range':
-            value = [str(v) for v in value]
+            value = [smart_unicode(v) for v in value]
         else:
-            value = str(value)
+            value = smart_unicode(value)
         return Field.get_db_prep_lookup(self, lookup_type, value)
 
     def get_manipulator_field_objs(self):
@@ -594,7 +595,7 @@ class DecimalField(Field):
         try:
             return decimal.Decimal(value)
         except decimal.InvalidOperation:
-            raise validators.ValidationError, gettext("This value must be a decimal number.")
+            raise validators.ValidationError, ugettext("This value must be a decimal number.")
 
     def _format(self, value):
         if isinstance(value, basestring):
@@ -615,7 +616,7 @@ class DecimalField(Field):
         if value < 0:
             num_chars += 1
 
-        return "%.*f" % (self.decimal_places, value)
+        return u"%.*f" % (self.decimal_places, value)
 
     def get_db_prep_save(self, value):
         if value is not None:
@@ -677,7 +678,7 @@ class FileField(Field):
                         self.always_test = True
                     def __call__(self, field_data, all_data):
                         if not all_data.get(self.other_file_field_name, False):
-                            c = validators.RequiredIfOtherFieldsGiven(self.other_field_names, gettext_lazy("This field is required."))
+                            c = validators.RequiredIfOtherFieldsGiven(self.other_field_names, ugettext_lazy("This field is required."))
                             c(field_data, all_data)
                 # First, get the core fields, if any.
                 core_field_names = []
@@ -688,7 +689,7 @@ class FileField(Field):
                 if core_field_names:
                     field_list[0].validator_list.append(RequiredFileField(core_field_names, field_list[1].field_name))
             else:
-                v = validators.RequiredIfOtherFieldNotGiven(field_list[1].field_name, gettext_lazy("This field is required."))
+                v = validators.RequiredIfOtherFieldNotGiven(field_list[1].field_name, ugettext_lazy("This field is required."))
                 v.always_test = True
                 field_list[0].validator_list.append(v)
                 field_list[0].is_required = field_list[1].is_required = False
@@ -822,7 +823,7 @@ class NullBooleanField(Field):
         if value in ('None'): return None
         if value in ('t', 'True', '1'): return True
         if value in ('f', 'False', '0'): return False
-        raise validators.ValidationError, gettext("This value must be either None, True or False.")
+        raise validators.ValidationError, _("This value must be either None, True or False.")
 
     def get_manipulator_field_objs(self):
         return [oldforms.NullBooleanField]
@@ -887,9 +888,9 @@ class TimeField(Field):
             def prep(value):
                 if isinstance(value, datetime.time):
                     value = datetime.datetime.combine(datetime.date(1900, 1, 1), value)
-                return str(value)
+                return smart_unicode(value)
         else:
-            prep = str
+            prep = smart_unicode
         if lookup_type == 'range':
             value = [prep(v) for v in value]
         else:
@@ -919,7 +920,7 @@ class TimeField(Field):
                 elif isinstance(value, basestring):
                     value = datetime.datetime(*(time.strptime(value, '%H:%M:%S')[:6]))
             else:
-                value = str(value)
+                value = smart_unicode(value)
         return Field.get_db_prep_save(self, value)
 
     def get_manipulator_field_objs(self):

django/db/models/fields/related.py

@@ -3,8 +3,9 @@ from django.db.models import signals, get_model
 from django.db.models.fields import AutoField, Field, IntegerField, get_ul_class
 from django.db.models.related import RelatedObject
 from django.utils.text import capfirst
-from django.utils.translation import gettext_lazy, string_concat, ngettext
+from django.utils.translation import ugettext_lazy, string_concat, ungettext, ugettext as _
 from django.utils.functional import curry
+from django.utils.encoding import smart_unicode
 from django.core import validators
 from django import oldforms
 from django import newforms as forms
@@ -637,9 +638,9 @@ class ManyToManyField(RelatedField, Field):
         Field.__init__(self, **kwargs)
 
         if self.rel.raw_id_admin:
-            msg = gettext_lazy('Separate multiple IDs with commas.')
+            msg = ugettext_lazy('Separate multiple IDs with commas.')
         else:
-            msg = gettext_lazy('Hold down "Control", or "Command" on a Mac, to select more than one.')
+            msg = ugettext_lazy('Hold down "Control", or "Command" on a Mac, to select more than one.')
         self.help_text = string_concat(self.help_text, ' ', msg)
 
     def get_manipulator_field_objs(self):
@@ -686,7 +687,7 @@ class ManyToManyField(RelatedField, Field):
         objects = mod._default_manager.in_bulk(pks)
         if len(objects) != len(pks):
             badkeys = [k for k in pks if k not in objects]
-            raise validators.ValidationError, ngettext("Please enter valid %(self)s IDs. The value %(value)r is invalid.",
+            raise validators.ValidationError, ungettext("Please enter valid %(self)s IDs. The value %(value)r is invalid.",
                     "Please enter valid %(self)s IDs. The values %(value)r are invalid.", len(badkeys)) % {
                 'self': self.verbose_name,
                 'value': len(badkeys) == 1 and badkeys[0] or tuple(badkeys),
@@ -697,7 +698,7 @@ class ManyToManyField(RelatedField, Field):
         if obj:
             instance_ids = [instance._get_pk_val() for instance in getattr(obj, self.name).all()]
             if self.rel.raw_id_admin:
-                new_data[self.name] = ",".join([str(id) for id in instance_ids])
+                new_data[self.name] = u",".join([smart_unicode(id) for id in instance_ids])
             else:
                 new_data[self.name] = instance_ids
         else:

django/db/models/manipulators.py

@@ -7,6 +7,8 @@ from django.db.models import signals
 from django.utils.functional import curry
 from django.utils.datastructures import DotExpandedDict
 from django.utils.text import capfirst
+from django.utils.encoding import smart_str
+from django.utils.translation import ugettext as _
 import types
 
 def add_manipulators(sender):
@@ -111,7 +113,7 @@ class AutomaticManipulator(oldforms.Manipulator):
         if self.change:
             self.fields_added, self.fields_changed, self.fields_deleted = [], [], []
             for f in self.opts.fields:
-                if not f.primary_key and str(getattr(self.original_object, f.attname)) != str(getattr(new_object, f.attname)):
+                if not f.primary_key and smart_str(getattr(self.original_object, f.attname)) != smart_str(getattr(new_object, f.attname)):
                     self.fields_changed.append(f.verbose_name)
 
         # Save many-to-many objects. Example: Set sites for a poll.
@@ -211,7 +213,7 @@ class AutomaticManipulator(oldforms.Manipulator):
                                 self.fields_added.append('%s "%s"' % (related.opts.verbose_name, new_rel_obj))
                             else:
                                 for f in related.opts.fields:
-                                    if not f.primary_key and f != related.field and str(getattr(old_rel_obj, f.attname)) != str(getattr(new_rel_obj, f.attname)):
+                                    if not f.primary_key and f != related.field and smart_str(getattr(old_rel_obj, f.attname)) != smart_str(getattr(new_rel_obj, f.attname)):
                                         self.fields_changed.append('%s for %s "%s"' % (f.verbose_name, related.opts.verbose_name, new_rel_obj))
 
                         # Save many-to-many objects.

django/db/models/options.py

@@ -5,6 +5,8 @@ from django.db.models.fields import AutoField, FieldDoesNotExist
 from django.db.models.loading import get_models
 from django.db.models.query import orderlist2sql
 from django.db.models import Manager
+from django.utils.translation import activate, deactivate_all, get_language, string_concat
+from django.utils.encoding import force_unicode, smart_str
 from bisect import bisect
 import re
 
@@ -42,6 +44,7 @@ class Options(object):
         self.object_name = cls.__name__
         self.module_name = self.object_name.lower()
         self.verbose_name = get_verbose_name(self.object_name)
+
         # Next, apply any overridden values from 'class Meta'.
         if self.meta:
             meta_attrs = self.meta.__dict__
@@ -51,12 +54,12 @@ class Options(object):
                 setattr(self, attr_name, meta_attrs.pop(attr_name, getattr(self, attr_name)))
             # verbose_name_plural is a special case because it uses a 's'
             # by default.
-            setattr(self, 'verbose_name_plural', meta_attrs.pop('verbose_name_plural', self.verbose_name + 's'))
+            setattr(self, 'verbose_name_plural', meta_attrs.pop('verbose_name_plural', string_concat(self.verbose_name, 's')))
             # Any leftover attributes must be invalid.
             if meta_attrs != {}:
                 raise TypeError, "'class Meta' got invalid attribute(s): %s" % ','.join(meta_attrs.keys())
         else:
-            self.verbose_name_plural = self.verbose_name + 's'
+            self.verbose_name_plural = string_concat(self.verbose_name, 's')
         del self.meta
 
     def _prepare(self, model):
@@ -95,7 +98,20 @@ class Options(object):
         return '<Options for %s>' % self.object_name
 
     def __str__(self):
-        return "%s.%s" % (self.app_label, self.module_name)
+        return "%s.%s" % (smart_str(self.app_label), smart_str(self.module_name))
+
+    def verbose_name_raw(self):
+        """
+        There are a few places where the untranslated verbose name is needed
+        (so that we get the same value regardless of currently active
+        locale).
+        """
+        lang = get_language()
+        deactivate_all()
+        raw = force_unicode(self.verbose_name)
+        activate(lang)
+        return raw
+    verbose_name_raw = property(verbose_name_raw)
 
     def get_field(self, name, many_to_many=True):
         "Returns the requested field by name. Raises FieldDoesNotExist on error."

django/db/models/query.py

@@ -4,6 +4,7 @@ from django.db.models.fields import DateField, FieldDoesNotExist
 from django.db.models import signals, loading
 from django.dispatch import dispatcher
 from django.utils.datastructures import SortedDict
+from django.utils.encoding import smart_unicode
 from django.contrib.contenttypes import generic
 import datetime
 import operator
@@ -52,7 +53,7 @@ def handle_legacy_orderlist(order_list):
         return order_list
     else:
         import warnings
-        new_order_list = [LEGACY_ORDERING_MAPPING[j.upper()].replace('_', str(i)) for i, j in order_list]
+        new_order_list = [LEGACY_ORDERING_MAPPING[j.upper()].replace('_', smart_unicode(i)) for i, j in order_list]
         warnings.warn("%r ordering syntax is deprecated. Use %r instead." % (order_list, new_order_list), DeprecationWarning)
         return new_order_list
 

django/http/__init__.py

@@ -1,8 +1,9 @@
 import os
 from Cookie import SimpleCookie
 from pprint import pformat
-from urllib import urlencode, quote
+from urllib import urlencode
 from django.utils.datastructures import MultiValueDict
+from django.utils.encoding import smart_str, iri_to_uri, force_unicode
 
 RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
 
@@ -17,6 +18,10 @@ class Http404(Exception):
 
 class HttpRequest(object):
     "A basic HTTP request"
+
+    # The encoding used in GET/POST dicts. None means use default setting.
+    _encoding = None
+
     def __init__(self):
         self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {}
         self.path = ''
@@ -42,14 +47,31 @@ class HttpRequest(object):
     def is_secure(self):
         return os.environ.get("HTTPS") == "on"
 
+    def _set_encoding(self, val):
+        """
+        Sets the encoding used for GET/POST accesses. If the GET or POST
+        dictionary has already been created, it is removed and recreated on the
+        next access (so that it is decoded correctly).
+        """
+        self._encoding = val
+        if hasattr(self, '_get'):
+            del self._get
+        if hasattr(self, '_post'):
+            del self._post
+
+    def _get_encoding(self):
+        return self._encoding
+
+    encoding = property(_get_encoding, _set_encoding)
+
 def parse_file_upload(header_dict, post_data):
-    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
+    "Returns a tuple of (POST QueryDict, FILES MultiValueDict)"
     import email, email.Message
     from cgi import parse_header
     raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
     raw_message += '\r\n\r\n' + post_data
     msg = email.message_from_string(raw_message)
-    POST = MultiValueDict()
+    POST = QueryDict('', mutable=True)
     FILES = MultiValueDict()
     for submessage in msg.get_payload():
         if submessage and isinstance(submessage, email.Message.Message):
@@ -74,13 +96,25 @@ def parse_file_upload(header_dict, post_data):
     return POST, FILES
 
 class QueryDict(MultiValueDict):
-    """A specialized MultiValueDict that takes a query string when initialized.
+    """
-    This is immutable unless you create a copy of it."""
+    A specialized MultiValueDict that takes a query string when initialized.
-    def __init__(self, query_string, mutable=False):
+    This is immutable unless you create a copy of it.
+
+    Values retrieved from this class are converted from the given encoding
+    (DEFAULT_CHARSET by default) to unicode.
+    """
+    def __init__(self, query_string, mutable=False, encoding=None):
         MultiValueDict.__init__(self)
+        if not encoding:
+            # *Important*: do not import settings any earlier because of note
+            # in core.handlers.modpython.
+            from django.conf import settings
+            self.encoding = settings.DEFAULT_CHARSET
+        else:
+            self.encoding = encoding
         self._mutable = True
         for key, value in parse_qsl((query_string or ''), True): # keep_blank_values=True
-            self.appendlist(key, value)
+            self.appendlist(force_unicode(key, errors='replace'), force_unicode(value, errors='replace'))
         self._mutable = mutable
 
     def _assert_mutable(self):
@@ -89,6 +123,8 @@ class QueryDict(MultiValueDict):
 
     def __setitem__(self, key, value):
         self._assert_mutable()
+        key = str_to_unicode(key, self.encoding)
+        value = str_to_unicode(value, self.encoding)
         MultiValueDict.__setitem__(self, key, value)
 
     def __delitem__(self, key):
@@ -111,15 +147,27 @@ class QueryDict(MultiValueDict):
 
     def setlist(self, key, list_):
         self._assert_mutable()
+        key = str_to_unicode(key, self.encoding)
+        list_ = [str_to_unicode(elt, self.encoding) for elt in list_]
         MultiValueDict.setlist(self, key, list_)
 
+    def setlistdefault(self, key, default_list=()):
+        self._assert_mutable()
+        if key not in self:
+            self.setlist(key, default_list)
+        return MultiValueDict.getlist(self, key)
+
     def appendlist(self, key, value):
         self._assert_mutable()
+        key = str_to_unicode(key, self.encoding)
+        value = str_to_unicode(value, self.encoding)
         MultiValueDict.appendlist(self, key, value)
 
     def update(self, other_dict):
         self._assert_mutable()
-        MultiValueDict.update(self, other_dict)
+        f = lambda s: str_to_unicode(s, self.encoding)
+        d = dict([(f(k), f(v)) for k, v in other_dict.items()])
+        MultiValueDict.update(self, d)
 
     def pop(self, key, *args):
         self._assert_mutable()
@@ -133,9 +181,11 @@ class QueryDict(MultiValueDict):
         self._assert_mutable()
         MultiValueDict.clear(self)
 
-    def setdefault(self, *args):
+    def setdefault(self, key, default=None):
         self._assert_mutable()
-        return MultiValueDict.setdefault(self, *args)
+        key = str_to_unicode(key, self.encoding)
+        default = str_to_unicode(default, self.encoding)
+        return MultiValueDict.setdefault(self, key, default)
 
     def copy(self):
         "Returns a mutable copy of this object."
@@ -144,7 +194,8 @@ class QueryDict(MultiValueDict):
     def urlencode(self):
         output = []
         for k, list_ in self.lists():
-            output.extend([urlencode({k: v}) for v in list_])
+            k = smart_str(k, self.encoding)
+            output.extend([urlencode({k: smart_str(v, self.encoding)}) for v in list_])
         return '&'.join(output)
 
 def parse_cookie(cookie):
@@ -221,9 +272,7 @@ class HttpResponse(object):
         self.cookies[key]['max-age'] = 0
 
     def _get_content(self):
-        content = ''.join(self._container)
+        content = smart_str(''.join(self._container), self._charset)
-        if isinstance(content, unicode):
-            content = content.encode(self._charset)
         return content
 
     def _set_content(self, value):
@@ -266,14 +315,14 @@ class HttpResponseRedirect(HttpResponse):
 
     def __init__(self, redirect_to):
         HttpResponse.__init__(self)
-        self['Location'] = quote(redirect_to, safe=RESERVED_CHARS)
+        self['Location'] = iri_to_uri(redirect_to)
 
 class HttpResponsePermanentRedirect(HttpResponse):
     status_code = 301
 
     def __init__(self, redirect_to):
         HttpResponse.__init__(self)
-        self['Location'] = quote(redirect_to, safe=RESERVED_CHARS)
+        self['Location'] = iri_to_uri(redirect_to)
 
 class HttpResponseNotModified(HttpResponse):
     status_code = 304
@@ -312,3 +361,20 @@ def get_host(request):
     if not host:
         host = request.META.get('HTTP_HOST', '')
     return host
+
+# It's neither necessary nor appropriate to use
+# django.utils.encoding.smart_unicode for parsing URLs and form inputs. Thus,
+# this slightly more restricted function.
+def str_to_unicode(s, encoding):
+    """
+    Convert basestring objects to unicode, using the given encoding. Illegaly
+    encoded input characters are replaced with Unicode "unknown" codepoint
+    (\ufffd).
+
+    Returns any non-basestring objects without change.
+    """
+    if isinstance(s, str):
+        return unicode(s, encoding, 'replace')
+    else:
+        return s
+

django/newforms/fields.py

@@ -6,7 +6,7 @@ import datetime
 import re
 import time
 
-from django.utils.translation import gettext
+from django.utils.translation import ugettext
 from django.utils.encoding import smart_unicode
 
 from util import ErrorList, ValidationError
@@ -84,7 +84,7 @@ class Field(object):
         Raises ValidationError for any errors.
         """
         if self.required and value in EMPTY_VALUES:
-            raise ValidationError(gettext(u'This field is required.'))
+            raise ValidationError(ugettext(u'This field is required.'))
         return value
 
     def widget_attrs(self, widget):
@@ -107,9 +107,9 @@ class CharField(Field):
             return u''
         value = smart_unicode(value)
         if self.max_length is not None and len(value) > self.max_length:
-            raise ValidationError(gettext(u'Ensure this value has at most %d characters.') % self.max_length)
+            raise ValidationError(ugettext(u'Ensure this value has at most %d characters.') % self.max_length)
         if self.min_length is not None and len(value) < self.min_length:
-            raise ValidationError(gettext(u'Ensure this value has at least %d characters.') % self.min_length)
+            raise ValidationError(ugettext(u'Ensure this value has at least %d characters.') % self.min_length)
         return value
 
     def widget_attrs(self, widget):
@@ -132,11 +132,11 @@ class IntegerField(Field):
         try:
             value = int(value)
         except (ValueError, TypeError):
-            raise ValidationError(gettext(u'Enter a whole number.'))
+            raise ValidationError(ugettext(u'Enter a whole number.'))
         if self.max_value is not None and value > self.max_value:
-            raise ValidationError(gettext(u'Ensure this value is less than or equal to %s.') % self.max_value)
+            raise ValidationError(ugettext(u'Ensure this value is less than or equal to %s.') % self.max_value)
         if self.min_value is not None and value < self.min_value:
-            raise ValidationError(gettext(u'Ensure this value is greater than or equal to %s.') % self.min_value)
+            raise ValidationError(ugettext(u'Ensure this value is greater than or equal to %s.') % self.min_value)
         return value
 
 class FloatField(Field):
@@ -155,11 +155,11 @@ class FloatField(Field):
         try:
             value = float(value)
         except (ValueError, TypeError):
-            raise ValidationError(gettext('Enter a number.'))
+            raise ValidationError(ugettext('Enter a number.'))
         if self.max_value is not None and value > self.max_value:
-            raise ValidationError(gettext('Ensure this value is less than or equal to %s.') % self.max_value)
+            raise ValidationError(ugettext('Ensure this value is less than or equal to %s.') % self.max_value)
         if self.min_value is not None and value < self.min_value:
-            raise ValidationError(gettext('Ensure this value is greater than or equal to %s.') % self.min_value)
+            raise ValidationError(ugettext('Ensure this value is greater than or equal to %s.') % self.min_value)
         return value
 
 decimal_re = re.compile(r'^-?(?P<digits>\d+)(\.(?P<decimals>\d+))?$')
@@ -183,21 +183,21 @@ class DecimalField(Field):
         value = value.strip()
         match = decimal_re.search(value)
         if not match:
-            raise ValidationError(gettext('Enter a number.'))
+            raise ValidationError(ugettext('Enter a number.'))
         else:
             value = Decimal(value)
         digits = len(match.group('digits') or '')
         decimals = len(match.group('decimals') or '')
         if self.max_value is not None and value > self.max_value:
-            raise ValidationError(gettext('Ensure this value is less than or equal to %s.') % self.max_value)
+            raise ValidationError(ugettext('Ensure this value is less than or equal to %s.') % self.max_value)
         if self.min_value is not None and value < self.min_value:
-            raise ValidationError(gettext('Ensure this value is greater than or equal to %s.') % self.min_value)
+            raise ValidationError(ugettext('Ensure this value is greater than or equal to %s.') % self.min_value)
         if self.max_digits is not None and (digits + decimals) > self.max_digits:
-            raise ValidationError(gettext('Ensure that there are no more than %s digits in total.') % self.max_digits)
+            raise ValidationError(ugettext('Ensure that there are no more than %s digits in total.') % self.max_digits)
         if self.decimal_places is not None and decimals > self.decimal_places:
-            raise ValidationError(gettext('Ensure that there are no more than %s decimal places.') % self.decimal_places)
+            raise ValidationError(ugettext('Ensure that there are no more than %s decimal places.') % self.decimal_places)
         if self.max_digits is not None and self.decimal_places is not None and digits > (self.max_digits - self.decimal_places):
-            raise ValidationError(gettext('Ensure that there are no more than %s digits before the decimal point.') % (self.max_digits - self.decimal_places))
+            raise ValidationError(ugettext('Ensure that there are no more than %s digits before the decimal point.') % (self.max_digits - self.decimal_places))
         return value
 
 DEFAULT_DATE_INPUT_FORMATS = (
@@ -230,7 +230,7 @@ class DateField(Field):
                 return datetime.date(*time.strptime(value, format)[:3])
             except ValueError:
                 continue
-        raise ValidationError(gettext(u'Enter a valid date.'))
+        raise ValidationError(ugettext(u'Enter a valid date.'))
 
 DEFAULT_TIME_INPUT_FORMATS = (
     '%H:%M:%S',     # '14:30:59'
@@ -257,7 +257,7 @@ class TimeField(Field):
                 return datetime.time(*time.strptime(value, format)[3:6])
             except ValueError:
                 continue
-        raise ValidationError(gettext(u'Enter a valid time.'))
+        raise ValidationError(ugettext(u'Enter a valid time.'))
 
 DEFAULT_DATETIME_INPUT_FORMATS = (
     '%Y-%m-%d %H:%M:%S',     # '2006-10-25 14:30:59'
@@ -293,7 +293,7 @@ class DateTimeField(Field):
                 return datetime.datetime(*time.strptime(value, format)[:6])
             except ValueError:
                 continue
-        raise ValidationError(gettext(u'Enter a valid date/time.'))
+        raise ValidationError(ugettext(u'Enter a valid date/time.'))
 
 class RegexField(Field):
     def __init__(self, regex, max_length=None, min_length=None, error_message=None, *args, **kwargs):
@@ -307,7 +307,7 @@ class RegexField(Field):
             regex = re.compile(regex)
         self.regex = regex
         self.max_length, self.min_length = max_length, min_length
-        self.error_message = error_message or gettext(u'Enter a valid value.')
+        self.error_message = error_message or ugettext(u'Enter a valid value.')
 
     def clean(self, value):
         """
@@ -321,9 +321,9 @@ class RegexField(Field):
         if value == u'':
             return value
         if self.max_length is not None and len(value) > self.max_length:
-            raise ValidationError(gettext(u'Ensure this value has at most %d characters.') % self.max_length)
+            raise ValidationError(ugettext(u'Ensure this value has at most %d characters.') % self.max_length)
         if self.min_length is not None and len(value) < self.min_length:
-            raise ValidationError(gettext(u'Ensure this value has at least %d characters.') % self.min_length)
+            raise ValidationError(ugettext(u'Ensure this value has at least %d characters.') % self.min_length)
         if not self.regex.search(value):
             raise ValidationError(self.error_message)
         return value
@@ -336,7 +336,7 @@ email_re = re.compile(
 class EmailField(RegexField):
     def __init__(self, max_length=None, min_length=None, *args, **kwargs):
         RegexField.__init__(self, email_re, max_length, min_length,
-            gettext(u'Enter a valid e-mail address.'), *args, **kwargs)
+            ugettext(u'Enter a valid e-mail address.'), *args, **kwargs)
 
 url_re = re.compile(
     r'^https?://' # http:// or https://
@@ -354,7 +354,7 @@ except ImportError:
 class URLField(RegexField):
     def __init__(self, max_length=None, min_length=None, verify_exists=False,
             validator_user_agent=URL_VALIDATOR_USER_AGENT, *args, **kwargs):
-        super(URLField, self).__init__(url_re, max_length, min_length, gettext(u'Enter a valid URL.'), *args, **kwargs)
+        super(URLField, self).__init__(url_re, max_length, min_length, ugettext(u'Enter a valid URL.'), *args, **kwargs)
         self.verify_exists = verify_exists
         self.user_agent = validator_user_agent
 
@@ -376,9 +376,9 @@ class URLField(RegexField):
                 req = urllib2.Request(value, None, headers)
                 u = urllib2.urlopen(req)
             except ValueError:
-                raise ValidationError(gettext(u'Enter a valid URL.'))
+                raise ValidationError(ugettext(u'Enter a valid URL.'))
             except: # urllib2.URLError, httplib.InvalidURL, etc.
-                raise ValidationError(gettext(u'This URL appears to be a broken link.'))
+                raise ValidationError(ugettext(u'This URL appears to be a broken link.'))
         return value
 
 class BooleanField(Field):
@@ -427,9 +427,9 @@ class ChoiceField(Field):
         value = smart_unicode(value)
         if value == u'':
             return value
-        valid_values = set([str(k) for k, v in self.choices])
+        valid_values = set([smart_unicode(k) for k, v in self.choices])
         if value not in valid_values:
-            raise ValidationError(gettext(u'Select a valid choice. That choice is not one of the available choices.'))
+            raise ValidationError(ugettext(u'Select a valid choice. That choice is not one of the available choices.'))
         return value
 
 class MultipleChoiceField(ChoiceField):
@@ -441,11 +441,11 @@ class MultipleChoiceField(ChoiceField):
         Validates that the input is a list or tuple.
         """
         if self.required and not value:
-            raise ValidationError(gettext(u'This field is required.'))
+            raise ValidationError(ugettext(u'This field is required.'))
         elif not self.required and not value:
             return []
         if not isinstance(value, (list, tuple)):
-            raise ValidationError(gettext(u'Enter a list of values.'))
+            raise ValidationError(ugettext(u'Enter a list of values.'))
         new_value = []
         for val in value:
             val = smart_unicode(val)
@@ -454,7 +454,7 @@ class MultipleChoiceField(ChoiceField):
         valid_values = set([smart_unicode(k) for k, v in self.choices])
         for val in new_value:
             if val not in valid_values:
-                raise ValidationError(gettext(u'Select a valid choice. %s is not one of the available choices.') % val)
+                raise ValidationError(ugettext(u'Select a valid choice. %s is not one of the available choices.') % val)
         return new_value
 
 class ComboField(Field):
@@ -520,18 +520,18 @@ class MultiValueField(Field):
         if not value or isinstance(value, (list, tuple)):
             if not value or not [v for v in value if v not in EMPTY_VALUES]:
                 if self.required:
-                    raise ValidationError(gettext(u'This field is required.'))
+                    raise ValidationError(ugettext(u'This field is required.'))
                 else:
                     return self.compress([])
         else:
-            raise ValidationError(gettext(u'Enter a list of values.'))
+            raise ValidationError(ugettext(u'Enter a list of values.'))
         for i, field in enumerate(self.fields):
             try:
                 field_value = value[i]
             except IndexError:
                 field_value = None
             if self.required and field_value in EMPTY_VALUES:
-                raise ValidationError(gettext(u'This field is required.'))
+                raise ValidationError(ugettext(u'This field is required.'))
             try:
                 clean_data.append(field.clean(field_value))
             except ValidationError, e:
@@ -564,8 +564,8 @@ class SplitDateTimeField(MultiValueField):
             # Raise a validation error if time or date is empty
             # (possible if SplitDateTimeField has required=False).
             if data_list[0] in EMPTY_VALUES:
-                raise ValidationError(gettext(u'Enter a valid date.'))
+                raise ValidationError(ugettext(u'Enter a valid date.'))
             if data_list[1] in EMPTY_VALUES:
-                raise ValidationError(gettext(u'Enter a valid time.'))
+                raise ValidationError(ugettext(u'Enter a valid time.'))
             return datetime.datetime.combine(*data_list)
         return None

django/newforms/forms.py

@@ -6,7 +6,7 @@ import copy
 
 from django.utils.datastructures import SortedDict
 from django.utils.html import escape
-from django.utils.encoding import StrAndUnicode
+from django.utils.encoding import StrAndUnicode, smart_unicode, force_unicode
 
 from fields import Field
 from widgets import TextInput, Textarea
@@ -119,13 +119,13 @@ class BaseForm(StrAndUnicode):
             bf_errors = ErrorList([escape(error) for error in bf.errors]) # Escape and cache in local variable.
             if bf.is_hidden:
                 if bf_errors:
-                    top_errors.extend(['(Hidden field %s) %s' % (name, e) for e in bf_errors])
+                    top_errors.extend(['(Hidden field %s) %s' % (name, force_unicode(e)) for e in bf_errors])
                 hidden_fields.append(unicode(bf))
             else:
                 if errors_on_separate_row and bf_errors:
-                    output.append(error_row % bf_errors)
+                    output.append(error_row % force_unicode(bf_errors))
                 if bf.label:
-                    label = escape(bf.label)
+                    label = escape(force_unicode(bf.label))
                     # Only add a colon if the label does not end in punctuation.
                     if label[-1] not in ':?.!':
                         label += ':'
@@ -133,10 +133,10 @@ class BaseForm(StrAndUnicode):
                 else:
                     label = ''
                 if field.help_text:
-                    help_text = help_text_html % field.help_text
+                    help_text = help_text_html % force_unicode(field.help_text)
                 else:
                     help_text = u''
-                output.append(normal_row % {'errors': bf_errors, 'label': label, 'field': unicode(bf), 'help_text': help_text})
+                output.append(normal_row % {'errors': force_unicode(bf_errors), 'label': force_unicode(label), 'field': unicode(bf), 'help_text': help_text})
         if top_errors:
             output.insert(0, error_row % top_errors)
         if hidden_fields: # Insert any hidden fields in the last row.
@@ -314,8 +314,8 @@ class BoundField(StrAndUnicode):
         associated Form has specified auto_id. Returns an empty string otherwise.
         """
         auto_id = self.form.auto_id
-        if auto_id and '%s' in str(auto_id):
+        if auto_id and '%s' in smart_unicode(auto_id):
-            return str(auto_id) % self.html_name
+            return smart_unicode(auto_id) % self.html_name
         elif auto_id:
             return self.html_name
         return ''

django/newforms/models.py

@@ -3,7 +3,9 @@ Helper functions for creating Form classes from Django models
 and database field objects.
 """
 
-from django.utils.translation import gettext
+from django.utils.translation import ugettext
+from django.utils.encoding import smart_unicode
+
 
 from util import ValidationError
 from forms import BaseForm, SortedDictFromList
@@ -122,7 +124,7 @@ class QuerySetIterator(object):
         if self.empty_label is not None:
             yield (u"", self.empty_label)
         for obj in self.queryset:
-            yield (obj._get_pk_val(), str(obj))
+            yield (obj._get_pk_val(), smart_unicode(obj))
         # Clear the QuerySet cache if required.
         if not self.cache_choices:
             self.queryset._result_cache = None
@@ -169,7 +171,7 @@ class ModelChoiceField(ChoiceField):
         try:
             value = self.queryset.model._default_manager.get(pk=value)
         except self.queryset.model.DoesNotExist:
-            raise ValidationError(gettext(u'Select a valid choice. That choice is not one of the available choices.'))
+            raise ValidationError(ugettext(u'Select a valid choice. That choice is not one of the available choices.'))
         return value
 
 class ModelMultipleChoiceField(ModelChoiceField):
@@ -182,17 +184,17 @@ class ModelMultipleChoiceField(ModelChoiceField):
 
     def clean(self, value):
         if self.required and not value:
-            raise ValidationError(gettext(u'This field is required.'))
+            raise ValidationError(ugettext(u'This field is required.'))
         elif not self.required and not value:
             return []
         if not isinstance(value, (list, tuple)):
-            raise ValidationError(gettext(u'Enter a list of values.'))
+            raise ValidationError(ugettext(u'Enter a list of values.'))
         final_values = []
         for val in value:
             try:
                 obj = self.queryset.model._default_manager.get(pk=val)
             except self.queryset.model.DoesNotExist:
-                raise ValidationError(gettext(u'Select a valid choice. %s is not one of the available choices.') % val)
+                raise ValidationError(ugettext(u'Select a valid choice. %s is not one of the available choices.') % val)
             else:
                 final_values.append(obj)
         return final_values

django/newforms/util.py

@@ -1,5 +1,5 @@
 from django.utils.html import escape
-from django.utils.encoding import smart_unicode
+from django.utils.encoding import smart_unicode, StrAndUnicode
 
 def flatatt(attrs):
     """
@@ -10,36 +10,36 @@ def flatatt(attrs):
     """
     return u''.join([u' %s="%s"' % (k, escape(v)) for k, v in attrs.items()])
 
-class ErrorDict(dict):
+class ErrorDict(dict, StrAndUnicode):
     """
     A collection of errors that knows how to display itself in various formats.
 
     The dictionary keys are the field names, and the values are the errors.
     """
-    def __str__(self):
+    def __unicode__(self):
         return self.as_ul()
 
     def as_ul(self):
         if not self: return u''
-        return u'<ul class="errorlist">%s</ul>' % ''.join([u'<li>%s%s</li>' % (k, v) for k, v in self.items()])
+        return u'<ul class="errorlist">%s</ul>' % ''.join([u'<li>%s%s</li>' % (k, smart_unicode(v)) for k, v in self.items()])
 
     def as_text(self):
-        return u'\n'.join([u'* %s\n%s' % (k, u'\n'.join([u'  * %s' % i for i in v])) for k, v in self.items()])
+        return u'\n'.join([u'* %s\n%s' % (k, u'\n'.join([u'  * %s' % smart_unicode(i) for i in v])) for k, v in self.items()])
 
-class ErrorList(list):
+class ErrorList(list, StrAndUnicode):
     """
     A collection of errors that knows how to display itself in various formats.
     """
-    def __str__(self):
+    def __unicode__(self):
         return self.as_ul()
 
     def as_ul(self):
         if not self: return u''
-        return u'<ul class="errorlist">%s</ul>' % ''.join([u'<li>%s</li>' % e for e in self])
+        return u'<ul class="errorlist">%s</ul>' % ''.join([u'<li>%s</li>' % smart_unicode(e) for e in self])
 
     def as_text(self):
         if not self: return u''
-        return u'\n'.join([u'* %s' % e for e in self])
+        return u'\n'.join([u'* %s' % smart_unicode(e) for e in self])
 
 class ValidationError(Exception):
     def __init__(self, message):

django/newforms/widgets.py

@@ -10,8 +10,8 @@ except NameError:
 from itertools import chain
 from django.utils.datastructures import MultiValueDict
 from django.utils.html import escape
-from django.utils.translation import gettext
+from django.utils.translation import ugettext
-from django.utils.encoding import StrAndUnicode, smart_unicode
+from django.utils.encoding import StrAndUnicode, force_unicode
 from util import flatatt
 
 __all__ = (
@@ -77,7 +77,7 @@ class Input(Widget):
     def render(self, name, value, attrs=None):
         if value is None: value = ''
         final_attrs = self.build_attrs(attrs, type=self.input_type, name=name)
-        if value != '': final_attrs['value'] = smart_unicode(value) # Only add the 'value' attribute if a value is non-empty.
+        if value != '': final_attrs['value'] = force_unicode(value) # Only add the 'value' attribute if a value is non-empty.
         return u'<input%s />' % flatatt(final_attrs)
 
 class TextInput(Input):
@@ -111,7 +111,7 @@ class MultipleHiddenInput(HiddenInput):
     def render(self, name, value, attrs=None, choices=()):
         if value is None: value = []
         final_attrs = self.build_attrs(attrs, type=self.input_type, name=name)
-        return u'\n'.join([(u'<input%s />' % flatatt(dict(value=smart_unicode(v), **final_attrs))) for v in value])
+        return u'\n'.join([(u'<input%s />' % flatatt(dict(value=force_unicode(v), **final_attrs))) for v in value])
 
     def value_from_datadict(self, data, name):
         if isinstance(data, MultiValueDict):
@@ -130,7 +130,7 @@ class Textarea(Widget):
 
     def render(self, name, value, attrs=None):
         if value is None: value = ''
-        value = smart_unicode(value)
+        value = force_unicode(value)
         final_attrs = self.build_attrs(attrs, name=name)
         return u'<textarea%s>%s</textarea>' % (flatatt(final_attrs), escape(value))
 
@@ -150,7 +150,7 @@ class CheckboxInput(Widget):
         if result:
             final_attrs['checked'] = 'checked'
         if value not in ('', True, False, None):
-            final_attrs['value'] = smart_unicode(value) # Only add the 'value' attribute if a value is non-empty.
+            final_attrs['value'] = force_unicode(value) # Only add the 'value' attribute if a value is non-empty.
         return u'<input%s />' % flatatt(final_attrs)
 
 class Select(Widget):
@@ -165,11 +165,11 @@ class Select(Widget):
         if value is None: value = ''
         final_attrs = self.build_attrs(attrs, name=name)
         output = [u'<select%s>' % flatatt(final_attrs)]
-        str_value = smart_unicode(value) # Normalize to string.
+        str_value = force_unicode(value) # Normalize to string.
         for option_value, option_label in chain(self.choices, choices):
-            option_value = smart_unicode(option_value)
+            option_value = force_unicode(option_value)
             selected_html = (option_value == str_value) and u' selected="selected"' or ''
-            output.append(u'<option value="%s"%s>%s</option>' % (escape(option_value), selected_html, escape(smart_unicode(option_label))))
+            output.append(u'<option value="%s"%s>%s</option>' % (escape(option_value), selected_html, escape(force_unicode(option_label))))
         output.append(u'</select>')
         return u'\n'.join(output)
 
@@ -178,7 +178,7 @@ class NullBooleanSelect(Select):
     A Select Widget intended to be used with NullBooleanField.
     """
     def __init__(self, attrs=None):
-        choices = ((u'1', gettext('Unknown')), (u'2', gettext('Yes')), (u'3', gettext('No')))
+        choices = ((u'1', ugettext('Unknown')), (u'2', ugettext('Yes')), (u'3', ugettext('No')))
         super(NullBooleanSelect, self).__init__(attrs, choices)
 
     def render(self, name, value, attrs=None, choices=()):
@@ -202,11 +202,11 @@ class SelectMultiple(Widget):
         if value is None: value = []
         final_attrs = self.build_attrs(attrs, name=name)
         output = [u'<select multiple="multiple"%s>' % flatatt(final_attrs)]
-        str_values = set([smart_unicode(v) for v in value]) # Normalize to strings.
+        str_values = set([force_unicode(v) for v in value]) # Normalize to strings.
         for option_value, option_label in chain(self.choices, choices):
-            option_value = smart_unicode(option_value)
+            option_value = force_unicode(option_value)
             selected_html = (option_value in str_values) and ' selected="selected"' or ''
-            output.append(u'<option value="%s"%s>%s</option>' % (escape(option_value), selected_html, escape(smart_unicode(option_label))))
+            output.append(u'<option value="%s"%s>%s</option>' % (escape(option_value), selected_html, escape(force_unicode(option_label))))
         output.append(u'</select>')
         return u'\n'.join(output)
 
@@ -220,8 +220,8 @@ class RadioInput(StrAndUnicode):
     def __init__(self, name, value, attrs, choice, index):
         self.name, self.value = name, value
         self.attrs = attrs
-        self.choice_value = smart_unicode(choice[0])
+        self.choice_value = force_unicode(choice[0])
-        self.choice_label = smart_unicode(choice[1])
+        self.choice_label = force_unicode(choice[1])
         self.index = index
 
     def __unicode__(self):
@@ -254,13 +254,13 @@ class RadioFieldRenderer(StrAndUnicode):
 
     def __unicode__(self):
         "Outputs a <ul> for this set of radio fields."
-        return u'<ul>\n%s\n</ul>' % u'\n'.join([u'<li>%s</li>' % w for w in self])
+        return u'<ul>\n%s\n</ul>' % u'\n'.join([u'<li>%s</li>' % force_unicode(w) for w in self])
 
 class RadioSelect(Select):
     def render(self, name, value, attrs=None, choices=()):
         "Returns a RadioFieldRenderer instance rather than a Unicode string."
         if value is None: value = ''
-        str_value = smart_unicode(value) # Normalize to string.
+        str_value = force_unicode(value) # Normalize to string.
         final_attrs = self.build_attrs(attrs)
         return RadioFieldRenderer(name, str_value, final_attrs, list(chain(self.choices, choices)))
 
@@ -280,16 +280,16 @@ class CheckboxSelectMultiple(SelectMultiple):
         has_id = attrs and 'id' in attrs
         final_attrs = self.build_attrs(attrs, name=name)
         output = [u'<ul>']
-        str_values = set([smart_unicode(v) for v in value]) # Normalize to strings.
+        str_values = set([force_unicode(v) for v in value]) # Normalize to strings.
         for i, (option_value, option_label) in enumerate(chain(self.choices, choices)):
             # If an ID attribute was given, add a numeric index as a suffix,
             # so that the checkboxes don't all have the same ID attribute.
             if has_id:
                 final_attrs = dict(final_attrs, id='%s_%s' % (attrs['id'], i))
             cb = CheckboxInput(final_attrs, check_test=lambda value: value in str_values)
-            option_value = smart_unicode(option_value)
+            option_value = force_unicode(option_value)
             rendered_cb = cb.render(name, option_value)
-            output.append(u'<li><label>%s %s</label></li>' % (rendered_cb, escape(smart_unicode(option_label))))
+            output.append(u'<li><label>%s %s</label></li>' % (rendered_cb, escape(force_unicode(option_label))))
         output.append(u'</ul>')
         return u'\n'.join(output)
 

django/oldforms/__init__.py

@@ -2,7 +2,8 @@ from django.core import validators
 from django.core.exceptions import PermissionDenied
 from django.utils.html import escape
 from django.conf import settings
-from django.utils.translation import gettext, ngettext
+from django.utils.translation import ugettext, ungettext
+from django.utils.encoding import smart_unicode, force_unicode, smart_str
 
 FORM_FIELD_ID_PREFIX = 'id_'
 
@@ -66,7 +67,7 @@ class Manipulator(object):
                     errors.setdefault(field.field_name, []).extend(e.messages)
 
 #            if field.is_required and not new_data.get(field.field_name, False):
-#                errors.setdefault(field.field_name, []).append(gettext_lazy('This field is required.'))
+#                errors.setdefault(field.field_name, []).append(ugettext_lazy('This field is required.'))
 #                continue
 #            try:
 #                validator_list = field.validator_list
@@ -166,7 +167,11 @@ class FormFieldWrapper(object):
 
     def __str__(self):
         "Renders the field"
-        return str(self.formfield.render(self.data))
+        return unicode(self).encode('utf-8')
+
+    def __unicode__(self):
+        "Renders the field"
+        return force_unicode(self.formfield.render(self.data))
 
     def __repr__(self):
         return '<FormFieldWrapper for "%s">' % self.formfield.field_name
@@ -196,7 +201,10 @@ class FormFieldCollection(FormFieldWrapper):
         self.formfield_dict = formfield_dict
 
     def __str__(self):
-        return str(self.formfield_dict)
+        return unicode(self).encode('utf-8')
+
+    def __str__(self):
+        return unicode(self.formfield_dict)
 
     def __getitem__(self, template_key):
         "Look up field by template key; raise KeyError on failure"
@@ -294,8 +302,12 @@ class FormField(object):
     Subclasses should also implement a render(data) method, which is responsible
     for rending the form field in XHTML.
     """
+
     def __str__(self):
-        return self.render('')
+        return unicode(self).encode('utf-8')
+
+    def __unicode__(self):
+        return self.render(u'')
 
     def __repr__(self):
         return 'FormField "%s"' % self.field_name
@@ -354,7 +366,7 @@ class FormField(object):
     def get_validation_errors(self, new_data):
         errors = {}
         if self.is_required and not new_data.get(self.field_name, False):
-            errors.setdefault(self.field_name, []).append(gettext('This field is required.'))
+            errors.setdefault(self.field_name, []).append(ugettext('This field is required.'))
             return errors
         try:
             for validator in self.validator_list:
@@ -388,24 +400,22 @@ class TextField(FormField):
             self.member_name = member_name
 
     def isValidLength(self, data, form):
-        if data and self.maxlength and len(data.decode(settings.DEFAULT_CHARSET)) > self.maxlength:
+        if data and self.maxlength and len(smart_unicode(data)) > self.maxlength:
-            raise validators.ValidationError, ngettext("Ensure your text is less than %s character.",
+            raise validators.ValidationError, ungettext("Ensure your text is less than %s character.",
                 "Ensure your text is less than %s characters.", self.maxlength) % self.maxlength
 
     def hasNoNewlines(self, data, form):
         if data and '\n' in data:
-            raise validators.ValidationError, gettext("Line breaks are not allowed here.")
+            raise validators.ValidationError, ugettext("Line breaks are not allowed here.")
 
     def render(self, data):
         if data is None:
-            data = ''
+            data = u''
-        maxlength = ''
+        maxlength = u''
         if self.maxlength:
-            maxlength = 'maxlength="%s" ' % self.maxlength
+            maxlength = u'maxlength="%s" ' % self.maxlength
-        if isinstance(data, unicode):
+        return u'<input type="%s" id="%s" class="v%s%s" name="%s" size="%s" value="%s" %s/>' % \
-            data = data.encode(settings.DEFAULT_CHARSET)
+            (self.input_type, self.get_id(), self.__class__.__name__, self.is_required and u' required' or '',
-        return '<input type="%s" id="%s" class="v%s%s" name="%s" size="%s" value="%s" %s/>' % \
-            (self.input_type, self.get_id(), self.__class__.__name__, self.is_required and ' required' or '',
             self.field_name, self.length, escape(data), maxlength)
 
     def html2python(data):
@@ -428,10 +438,8 @@ class LargeTextField(TextField):
     def render(self, data):
         if data is None:
             data = ''
-        if isinstance(data, unicode):
+        return u'<textarea id="%s" class="v%s%s" name="%s" rows="%s" cols="%s">%s</textarea>' % \
-            data = data.encode(settings.DEFAULT_CHARSET)
+            (self.get_id(), self.__class__.__name__, self.is_required and u' required' or u'',
-        return '<textarea id="%s" class="v%s%s" name="%s" rows="%s" cols="%s">%s</textarea>' % \
-            (self.get_id(), self.__class__.__name__, self.is_required and ' required' or '',
             self.field_name, self.rows, self.cols, escape(data))
 
 class HiddenField(FormField):
@@ -441,7 +449,7 @@ class HiddenField(FormField):
         self.validator_list = validator_list[:]
 
     def render(self, data):
-        return '<input type="hidden" id="%s" name="%s" value="%s" />' % \
+        return u'<input type="hidden" id="%s" name="%s" value="%s" />' % \
             (self.get_id(), self.field_name, escape(data))
 
 class CheckboxField(FormField):
@@ -456,7 +464,7 @@ class CheckboxField(FormField):
         checked_html = ''
         if data or (data is '' and self.checked_by_default):
             checked_html = ' checked="checked"'
-        return '<input type="checkbox" id="%s" class="v%s" name="%s"%s />' % \
+        return u'<input type="checkbox" id="%s" class="v%s" name="%s"%s />' % \
             (self.get_id(), self.__class__.__name__,
             self.field_name, checked_html)
 
@@ -471,6 +479,7 @@ class SelectField(FormField):
     def __init__(self, field_name, choices=None, size=1, is_required=False, validator_list=None, member_name=None):
         if validator_list is None: validator_list = []
         if choices is None: choices = []
+        choices = [(k, smart_unicode(v, strings_only=True)) for k, v in choices]
         self.field_name = field_name
         # choices is a list of (value, human-readable key) tuples because order matters
         self.choices, self.size, self.is_required = choices, size, is_required
@@ -479,23 +488,23 @@ class SelectField(FormField):
             self.member_name = member_name
 
     def render(self, data):
-        output = ['<select id="%s" class="v%s%s" name="%s" size="%s">' % \
+        output = [u'<select id="%s" class="v%s%s" name="%s" size="%s">' % \
             (self.get_id(), self.__class__.__name__,
-             self.is_required and ' required' or '', self.field_name, self.size)]
+             self.is_required and u' required' or u'', self.field_name, self.size)]
-        str_data = str(data) # normalize to string
+        str_data = smart_unicode(data) # normalize to string
         for value, display_name in self.choices:
-            selected_html = ''
+            selected_html = u''
-            if str(value) == str_data:
+            if smart_unicode(value) == str_data:
-                selected_html = ' selected="selected"'
+                selected_html = u' selected="selected"'
-            output.append('    <option value="%s"%s>%s</option>' % (escape(value), selected_html, escape(display_name)))
+            output.append(u'    <option value="%s"%s>%s</option>' % (escape(value), selected_html, escape(display_name)))
-        output.append('  </select>')
+        output.append(u'  </select>')
-        return '\n'.join(output)
+        return u'\n'.join(output)
 
     def isValidChoice(self, data, form):
-        str_data = str(data)
+        str_data = smart_unicode(data)
-        str_choices = [str(item[0]) for item in self.choices]
+        str_choices = [smart_str(item[0]) for item in self.choices]
         if str_data not in str_choices:
-            raise validators.ValidationError, gettext("Select a valid choice; '%(data)s' is not in %(choices)s.") % {'data': str_data, 'choices': str_choices}
+            raise validators.ValidationError, ugettext("Select a valid choice; '%(data)s' is not in %(choices)s.") % {'data': str_data, 'choices': str_choices}
 
 class NullSelectField(SelectField):
     "This SelectField converts blank fields to None"
@@ -509,6 +518,7 @@ class RadioSelectField(FormField):
     def __init__(self, field_name, choices=None, ul_class='', is_required=False, validator_list=None, member_name=None):
         if validator_list is None: validator_list = []
         if choices is None: choices = []
+        choices = [(k, smart_unicode(v)) for k, v in choices]
         self.field_name = field_name
         # choices is a list of (value, human-readable key) tuples because order matters
         self.choices, self.is_required = choices, is_required
@@ -520,7 +530,7 @@ class RadioSelectField(FormField):
     def render(self, data):
         """
         Returns a special object, RadioFieldRenderer, that is iterable *and*
-        has a default str() rendered output.
+        has a default unicode() rendered output.
 
         This allows for flexible use in templates. You can just use the default
         rendering:
@@ -537,44 +547,44 @@ class RadioSelectField(FormField):
         class RadioFieldRenderer:
             def __init__(self, datalist, ul_class):
                 self.datalist, self.ul_class = datalist, ul_class
-            def __str__(self):
+            def __unicode__(self):
-                "Default str() output for this radio field -- a <ul>"
+                "Default unicode() output for this radio field -- a <ul>"
-                output = ['<ul%s>' % (self.ul_class and ' class="%s"' % self.ul_class or '')]
+                output = [u'<ul%s>' % (self.ul_class and u' class="%s"' % self.ul_class or u'')]
-                output.extend(['<li>%s %s</li>' % (d['field'], d['label']) for d in self.datalist])
+                output.extend([u'<li>%s %s</li>' % (d['field'], d['label']) for d in self.datalist])
-                output.append('</ul>')
+                output.append(u'</ul>')
-                return ''.join(output)
+                return u''.join(output)
             def __iter__(self):
                 for d in self.datalist:
                     yield d
             def __len__(self):
                 return len(self.datalist)
         datalist = []
-        str_data = str(data) # normalize to string
+        str_data = smart_unicode(data) # normalize to string
         for i, (value, display_name) in enumerate(self.choices):
             selected_html = ''
-            if str(value) == str_data:
+            if smart_unicode(value) == str_data:
-                selected_html = ' checked="checked"'
+                selected_html = u' checked="checked"'
             datalist.append({
                 'value': value,
                 'name': display_name,
-                'field': '<input type="radio" id="%s" name="%s" value="%s"%s/>' % \
+                'field': u'<input type="radio" id="%s" name="%s" value="%s"%s/>' % \
-                    (self.get_id() + '_' + str(i), self.field_name, value, selected_html),
+                    (self.get_id() + u'_' + unicode(i), self.field_name, value, selected_html),
-                'label': '<label for="%s">%s</label>' % \
+                'label': u'<label for="%s">%s</label>' % \
-                    (self.get_id() + '_' + str(i), display_name),
+                    (self.get_id() + u'_' + unicode(i), display_name),
             })
         return RadioFieldRenderer(datalist, self.ul_class)
 
     def isValidChoice(self, data, form):
-        str_data = str(data)
+        str_data = smart_unicode(data)
-        str_choices = [str(item[0]) for item in self.choices]
+        str_choices = [smart_unicode(item[0]) for item in self.choices]
         if str_data not in str_choices:
-            raise validators.ValidationError, gettext("Select a valid choice; '%(data)s' is not in %(choices)s.") % {'data':str_data, 'choices':str_choices}
+            raise validators.ValidationError, ugettext("Select a valid choice; '%(data)s' is not in %(choices)s.") % {'data':str_data, 'choices':str_choices}
 
 class NullBooleanField(SelectField):
     "This SelectField provides 'Yes', 'No' and 'Unknown', mapping results to True, False or None"
     def __init__(self, field_name, is_required=False, validator_list=None):
         if validator_list is None: validator_list = []
-        SelectField.__init__(self, field_name, choices=[('1', _('Unknown')), ('2', _('Yes')), ('3', _('No'))],
+        SelectField.__init__(self, field_name, choices=[('1', ugettext('Unknown')), ('2', ugettext('Yes')), ('3', ugettext('No'))],
             is_required=is_required, validator_list=validator_list)
 
     def render(self, data):
@@ -590,24 +600,24 @@ class NullBooleanField(SelectField):
 class SelectMultipleField(SelectField):
     requires_data_list = True
     def render(self, data):
-        output = ['<select id="%s" class="v%s%s" name="%s" size="%s" multiple="multiple">' % \
+        output = [u'<select id="%s" class="v%s%s" name="%s" size="%s" multiple="multiple">' % \
-            (self.get_id(), self.__class__.__name__, self.is_required and ' required' or '',
+            (self.get_id(), self.__class__.__name__, self.is_required and u' required' or u'',
             self.field_name, self.size)]
-        str_data_list = map(str, data) # normalize to strings
+        str_data_list = map(smart_unicode, data) # normalize to strings
         for value, choice in self.choices:
-            selected_html = ''
+            selected_html = u''
-            if str(value) in str_data_list:
+            if smart_unicode(value) in str_data_list:
-                selected_html = ' selected="selected"'
+                selected_html = u' selected="selected"'
-            output.append('    <option value="%s"%s>%s</option>' % (escape(value), selected_html, escape(choice)))
+            output.append(u'    <option value="%s"%s>%s</option>' % (escape(value), selected_html, escape(choice)))
-        output.append('  </select>')
+        output.append(u'  </select>')
-        return '\n'.join(output)
+        return u'\n'.join(output)
 
     def isValidChoice(self, field_data, all_data):
         # data is something like ['1', '2', '3']
-        str_choices = [str(item[0]) for item in self.choices]
+        str_choices = [smart_unicode(item[0]) for item in self.choices]
-        for val in map(str, field_data):
+        for val in map(smart_unicode, field_data):
             if val not in str_choices:
-                raise validators.ValidationError, gettext("Select a valid choice; '%(data)s' is not in %(choices)s.") % {'data':val, 'choices':str_choices}
+                raise validators.ValidationError, ugettext("Select a valid choice; '%(data)s' is not in %(choices)s.") % {'data':val, 'choices':str_choices}
 
     def html2python(data):
         if data is None:
@@ -642,18 +652,18 @@ class CheckboxSelectMultipleField(SelectMultipleField):
         new_data.setlist(self.field_name, data_list)
 
     def render(self, data):
-        output = ['<ul%s>' % (self.ul_class and ' class="%s"' % self.ul_class or '')]
+        output = [u'<ul%s>' % (self.ul_class and u' class="%s"' % self.ul_class or u'')]
-        str_data_list = map(str, data) # normalize to strings
+        str_data_list = map(smart_unicode, data) # normalize to strings
         for value, choice in self.choices:
-            checked_html = ''
+            checked_html = u''
-            if str(value) in str_data_list:
+            if smart_unicode(value) in str_data_list:
-                checked_html = ' checked="checked"'
+                checked_html = u' checked="checked"'
-            field_name = '%s%s' % (self.field_name, value)
+            field_name = u'%s%s' % (self.field_name, value)
-            output.append('<li><input type="checkbox" id="%s" class="v%s" name="%s"%s value="on" /> <label for="%s">%s</label></li>' % \
+            output.append(u'<li><input type="checkbox" id="%s" class="v%s" name="%s"%s value="on" /> <label for="%s">%s</label></li>' % \
                 (self.get_id() + escape(value), self.__class__.__name__, field_name, checked_html,
                 self.get_id() + escape(value), choice))
-        output.append('</ul>')
+        output.append(u'</ul>')
-        return '\n'.join(output)
+        return u'\n'.join(output)
 
 ####################
 # FILE UPLOADS     #
@@ -669,12 +679,12 @@ class FileUploadField(FormField):
         try:
             content = field_data['content']
         except TypeError:
-            raise validators.CriticalValidationError, gettext("No file was submitted. Check the encoding type on the form.")
+            raise validators.CriticalValidationError, ugettext("No file was submitted. Check the encoding type on the form.")
         if not content:
-            raise validators.CriticalValidationError, gettext("The submitted file is empty.")
+            raise validators.CriticalValidationError, ugettext("The submitted file is empty.")
 
     def render(self, data):
-        return '<input type="file" id="%s" class="v%s" name="%s" />' % \
+        return u'<input type="file" id="%s" class="v%s" name="%s" />' % \
             (self.get_id(), self.__class__.__name__, self.field_name)
 
     def html2python(data):
@@ -727,7 +737,7 @@ class SmallIntegerField(IntegerField):
 
     def isSmallInteger(self, field_data, all_data):
         if not -32768 <= int(field_data) <= 32767:
-            raise validators.CriticalValidationError, gettext("Enter a whole number between -32,768 and 32,767.")
+            raise validators.CriticalValidationError, ugettext("Enter a whole number between -32,768 and 32,767.")
 
 class PositiveIntegerField(IntegerField):
     def __init__(self, field_name, length=10, maxlength=None, is_required=False, validator_list=None):
@@ -737,7 +747,7 @@ class PositiveIntegerField(IntegerField):
 
     def isPositive(self, field_data, all_data):
         if int(field_data) < 0:
-            raise validators.CriticalValidationError, gettext("Enter a positive number.")
+            raise validators.CriticalValidationError, ugettext("Enter a positive number.")
 
 class PositiveSmallIntegerField(IntegerField):
     def __init__(self, field_name, length=5, maxlength=None, is_required=False, validator_list=None):
@@ -747,7 +757,7 @@ class PositiveSmallIntegerField(IntegerField):
 
     def isPositiveSmall(self, field_data, all_data):
         if not 0 <= int(field_data) <= 32767:
-            raise validators.CriticalValidationError, gettext("Enter a whole number between 0 and 32,767.")
+            raise validators.CriticalValidationError, ugettext("Enter a whole number between 0 and 32,767.")
 
 class FloatField(TextField):
     def __init__(self, field_name, is_required=False, validator_list=None): 
@@ -1005,9 +1015,9 @@ class CommaSeparatedIntegerField(TextField):
 
     def render(self, data):
         if data is None:
-            data = ''
+            data = u''
         elif isinstance(data, (list, tuple)):
-            data = ','.join(data)
+            data = u','.join(data)
         return super(CommaSeparatedIntegerField, self).render(data)
 
 class RawIdAdminField(CommaSeparatedIntegerField):

django/template/__init__.py

@@ -58,8 +58,10 @@ import re
 from inspect import getargspec
 from django.conf import settings
 from django.template.context import Context, RequestContext, ContextPopException
-from django.utils.functional import curry
+from django.utils.functional import curry, Promise
 from django.utils.text import smart_split
+from django.utils.encoding import smart_unicode, force_unicode
+from django.utils.translation import ugettext as _
 
 __all__ = ('Template', 'Context', 'RequestContext', 'compile_string')
 
@@ -122,6 +124,9 @@ class TemplateSyntaxError(Exception):
 class TemplateDoesNotExist(Exception):
     pass
 
+class TemplateEncodingError(Exception):
+    pass
+
 class VariableDoesNotExist(Exception):
 
     def __init__(self, msg, params=()):
@@ -155,6 +160,10 @@ class StringOrigin(Origin):
 class Template(object):
     def __init__(self, template_string, origin=None, name='<Unknown Template>'):
         "Compilation stage"
+        try:
+            template_string = smart_unicode(template_string)
+        except UnicodeDecodeError:
+            raise TemplateEncodingError("Templates can only be constructed from unicode or UTF-8 strings.")
         if settings.TEMPLATE_DEBUG and origin == None:
             origin = StringOrigin(template_string)
             # Could do some crazy stack-frame stuff to record where this string
@@ -693,6 +702,13 @@ def resolve_variable(path, context):
                     else:
                         raise
             del bits[0]
+    if isinstance(current, (basestring, Promise)):
+        try:
+            current = force_unicode(current)
+        except UnicodeDecodeError:
+            # Failing to convert to unicode can happen sometimes (e.g. debug
+            # tracebacks). So we allow it in this particular instance.
+            pass
     return current
 
 class Node(object):
@@ -720,7 +736,7 @@ class NodeList(list):
                 bits.append(self.render_node(node, context))
             else:
                 bits.append(node)
-        return ''.join(bits)
+        return ''.join([force_unicode(b) for b in bits])
 
     def get_nodes_by_type(self, nodetype):
         "Return a list of all nodes of the given type"
@@ -730,7 +746,7 @@ class NodeList(list):
         return nodes
 
     def render_node(self, node, context):
-        return(node.render(context))
+        return node.render(context)
 
 class DebugNodeList(NodeList):
     def render_node(self, node, context):
@@ -765,32 +781,17 @@ class VariableNode(Node):
     def __repr__(self):
         return "<Variable Node: %s>" % self.filter_expression
 
-    def encode_output(self, output):
-        # Check type so that we don't run str() on a Unicode object
-        if not isinstance(output, basestring):
-            try:
-                return str(output)
-            except UnicodeEncodeError:
-                # If __str__() returns a Unicode object, convert it to bytestring.
-                return unicode(output).encode(settings.DEFAULT_CHARSET)
-        elif isinstance(output, unicode):
-            return output.encode(settings.DEFAULT_CHARSET)
-        else:
-            return output
-
     def render(self, context):
-        output = self.filter_expression.resolve(context)
+        return self.filter_expression.resolve(context)
-        return self.encode_output(output)
 
 class DebugVariableNode(VariableNode):
     def render(self, context):
         try:
-            output = self.filter_expression.resolve(context)
+            return self.filter_expression.resolve(context)
         except TemplateSyntaxError, e:
             if not hasattr(e, 'source'):
                 e.source = self.source
             raise
-        return self.encode_output(output)
 
 def generic_tag_compiler(params, defaults, name, node_class, parser, token):
     "Returns a template.Node subclass."

django/template/defaultfilters.py

@@ -2,7 +2,8 @@
 
 from django.template import resolve_variable, Library
 from django.conf import settings
-from django.utils.translation import gettext, ngettext
+from django.utils.translation import ugettext, ungettext
+from django.utils.encoding import force_unicode, smart_str, iri_to_uri
 import re
 import random as random_module
 
@@ -12,29 +13,18 @@ register = Library()
 # STRING DECORATOR    #
 #######################
 
-def smart_string(obj):
-    # FUTURE: Unicode strings should probably be normalized to a specific
-    # encoding and non-unicode strings should be converted to unicode too.
-#    if isinstance(obj, unicode):
-#        obj = obj.encode(settings.DEFAULT_CHARSET)
-#    else:
-#        obj = unicode(obj, settings.DEFAULT_CHARSET)
-    # FUTURE: Replace dumb string logic below with cool unicode logic above.
-    if not isinstance(obj, basestring):
-        obj = str(obj)
-    return obj
-
 def stringfilter(func):
     """
-    Decorator for filters which should only receive strings. The object passed
+    Decorator for filters which should only receive unicode objects. The object
-    as the first positional argument will be converted to a string.
+    passed as the first positional argument will be converted to a unicode
+    object.
     """
     def _dec(*args, **kwargs):
         if args:
             args = list(args)
-            args[0] = smart_string(args[0])
+            args[0] = force_unicode(args[0])
         return func(*args, **kwargs)
-        
+
     # Include a reference to the real function (used to check original
     # arguments by the template parser).
     _dec._decorated_function = getattr(func, '_decorated_function', func)
@@ -54,7 +44,7 @@ def capfirst(value):
     "Capitalizes the first character of the value"
     return value and value[0].upper() + value[1:]
 capfirst = stringfilter(capfirst)
- 
+
 def fix_ampersands(value):
     "Replaces ampersands with ``&`` entities"
     from django.utils.html import fix_ampersands
@@ -83,27 +73,32 @@ def floatformat(text, arg=-1):
     try:
         f = float(text)
     except ValueError:
-        return ''
+        return u''
     try:
         d = int(arg)
     except ValueError:
-        return smart_string(f)
+        return force_unicode(f)
     m = f - int(f)
     if not m and d < 0:
-        return '%d' % int(f)
+        return u'%d' % int(f)
     else:
-        formatstr = '%%.%df' % abs(d)
+        formatstr = u'%%.%df' % abs(d)
         return formatstr % f
 
+def iriencode(value):
+    "Escapes an IRI value for use in a URL"
+    return force_unicode(iri_to_uri(value))
+iriencode = stringfilter(iriencode)
+
 def linenumbers(value):
     "Displays text with line numbers"
     from django.utils.html import escape
-    lines = value.split('\n')
+    lines = value.split(u'\n')
     # Find the maximum width of the line count, for use with zero padding string format command
-    width = str(len(str(len(lines))))
+    width = unicode(len(unicode(len(lines))))
     for i, line in enumerate(lines):
-        lines[i] = ("%0" + width  + "d. %s") % (i + 1, escape(line))
+        lines[i] = (u"%0" + width  + u"d. %s") % (i + 1, escape(line))
-    return '\n'.join(lines)
+    return u'\n'.join(lines)
 linenumbers = stringfilter(linenumbers)
 
 def lower(value):
@@ -120,8 +115,13 @@ def make_list(value):
 make_list = stringfilter(make_list)
 
 def slugify(value):
-    "Converts to lowercase, removes non-alpha chars and converts spaces to hyphens"
+    """
-    value = re.sub('[^\w\s-]', '', value).strip().lower()
+    Normalizes string, converts to lowercase, removes non-alpha chars and
+    converts spaces to hyphens.
+    """
+    import unicodedata
+    value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore')
+    value = unicode(re.sub('[^\w\s-]', '', value).strip().lower())
     return re.sub('[-\s]+', '-', value)
 slugify = stringfilter(slugify)
 
@@ -135,9 +135,9 @@ def stringformat(value, arg):
     of Python string formatting
     """
     try:
-        return ("%" + str(arg)) % value
+        return (u"%" + unicode(arg)) % value
     except (ValueError, TypeError):
-        return ""
+        return u""
 
 def title(value):
     "Converts a string into titlecase"
@@ -155,8 +155,6 @@ def truncatewords(value, arg):
         length = int(arg)
     except ValueError: # invalid literal for int()
         return value # Fail silently.
-    if not isinstance(value, basestring):
-        value = str(value)
     return truncate_words(value, length)
 truncatewords = stringfilter(truncatewords)
 
@@ -171,8 +169,6 @@ def truncatewords_html(value, arg):
         length = int(arg)
     except ValueError: # invalid literal for int()
         return value # Fail silently.
-    if not isinstance(value, basestring):
-        value = str(value)
     return truncate_html_words(value, length)
 truncatewords_html = stringfilter(truncatewords_html)
 
@@ -183,10 +179,8 @@ upper = stringfilter(upper)
 
 def urlencode(value):
     "Escapes a value for use in a URL"
-    import urllib
+    from django.utils.http import urlquote
-    if not isinstance(value, basestring):
+    return urlquote(value)
-        value = str(value)
-    return urllib.quote(value)
 urlencode = stringfilter(urlencode)
 
 def urlize(value):
@@ -246,7 +240,7 @@ center = stringfilter(center)
 
 def cut(value, arg):
     "Removes all values of arg from the given string"
-    return value.replace(arg, '')
+    return value.replace(arg, u'')
 cut = stringfilter(cut)
 
 ###################
@@ -273,11 +267,11 @@ linebreaksbr = stringfilter(linebreaksbr)
 def removetags(value, tags):
     "Removes a space separated list of [X]HTML tags from the output"
     tags = [re.escape(tag) for tag in tags.split()]
-    tags_re = '(%s)' % '|'.join(tags)
+    tags_re = u'(%s)' % u'|'.join(tags)
-    starttag_re = re.compile(r'<%s(/?>|(\s+[^>]*>))' % tags_re)
+    starttag_re = re.compile(ur'<%s(/?>|(\s+[^>]*>))' % tags_re, re.U)
-    endtag_re = re.compile('</%s>' % tags_re)
+    endtag_re = re.compile(u'</%s>' % tags_re)
-    value = starttag_re.sub('', value)
+    value = starttag_re.sub(u'', value)
-    value = endtag_re.sub('', value)
+    value = endtag_re.sub(u'', value)
     return value
 removetags = stringfilter(removetags)
 
@@ -296,7 +290,7 @@ def dictsort(value, arg):
     Takes a list of dicts, returns that list sorted by the property given in
     the argument.
     """
-    decorated = [(resolve_variable('var.' + arg, {'var' : item}), item) for item in value]
+    decorated = [(resolve_variable(u'var.' + arg, {u'var' : item}), item) for item in value]
     decorated.sort()
     return [item[1] for item in decorated]
 
@@ -305,7 +299,7 @@ def dictsortreversed(value, arg):
     Takes a list of dicts, returns that list sorted in reverse order by the
     property given in the argument.
     """
-    decorated = [(resolve_variable('var.' + arg, {'var' : item}), item) for item in value]
+    decorated = [(resolve_variable(u'var.' + arg, {u'var' : item}), item) for item in value]
     decorated.sort()
     decorated.reverse()
     return [item[1] for item in decorated]
@@ -315,12 +309,12 @@ def first(value):
     try:
         return value[0]
     except IndexError:
-        return ''
+        return u''
 
 def join(value, arg):
     "Joins a list with a string, like Python's ``str.join(list)``"
     try:
-        return arg.join(map(smart_string, value))
+        return arg.join(map(force_unicode, value))
     except AttributeError: # fail silently but nicely
         return value
 
@@ -346,7 +340,7 @@ def slice_(value, arg):
     """
     try:
         bits = []
-        for x in arg.split(':'):
+        for x in arg.split(u':'):
             if len(x) == 0:
                 bits.append(None)
             else:
@@ -378,12 +372,12 @@ def unordered_list(value):
         </li>
     """
     def _helper(value, tabs):
-        indent = '\t' * tabs
+        indent = u'\t' * tabs
         if value[1]:
-            return '%s<li>%s\n%s<ul>\n%s\n%s</ul>\n%s</li>' % (indent, value[0], indent,
+            return u'%s<li>%s\n%s<ul>\n%s\n%s</ul>\n%s</li>' % (indent, force_unicode(value[0]), indent,
-                '\n'.join([_helper(v, tabs+1) for v in value[1]]), indent, indent)
+                u'\n'.join([_helper(v, tabs+1) for v in value[1]]), indent, indent)
         else:
-            return '%s<li>%s</li>' % (indent, value[0])
+            return u'%s<li>%s</li>' % (indent, force_unicode(value[0]))
     return _helper(value, 1)
 
 ###################
@@ -421,7 +415,7 @@ def date(value, arg=None):
     "Formats a date according to the given format"
     from django.utils.dateformat import format
     if not value:
-        return ''
+        return u''
     if arg is None:
         arg = settings.DATE_FORMAT
     return format(value, arg)
@@ -429,8 +423,8 @@ def date(value, arg=None):
 def time(value, arg=None):
     "Formats a time according to the given format"
     from django.utils.dateformat import time_format
-    if value in (None, ''):
+    if value in (None, u''):
-        return ''
+        return u''
     if arg is None:
         arg = settings.TIME_FORMAT
     return time_format(value, arg)
@@ -439,7 +433,7 @@ def timesince(value, arg=None):
     'Formats a date as the time since that date (i.e. "4 days, 6 hours")'
     from django.utils.timesince import timesince
     if not value:
-        return ''
+        return u''
     if arg:
         return timesince(arg, value)
     return timesince(value)
@@ -449,7 +443,7 @@ def timeuntil(value, arg=None):
     from django.utils.timesince import timesince
     from datetime import datetime
     if not value:
-        return ''
+        return u''
     if arg:
         return timesince(arg, value)
     return timesince(datetime.now(), value)
@@ -488,8 +482,8 @@ def yesno(value, arg=None):
     ==========  ======================  ==================================
     """
     if arg is None:
-        arg = gettext('yes,no,maybe')
+        arg = ugettext('yes,no,maybe')
-    bits = arg.split(',')
+    bits = arg.split(u',')
     if len(bits) < 2:
         return value # Invalid arg.
     try:
@@ -514,28 +508,28 @@ def filesizeformat(bytes):
     try:
         bytes = float(bytes)
     except TypeError:
-        return "0 bytes"
+        return u"0 bytes"
-        
-    if bytes < 1024:
-        return ngettext("%(size)d byte", "%(size)d bytes", bytes) % {'size': bytes}
-    if bytes < 1024 * 1024:
-        return gettext("%.1f KB") % (bytes / 1024)
-    if bytes < 1024 * 1024 * 1024:
-        return gettext("%.1f MB") % (bytes / (1024 * 1024))
-    return gettext("%.1f GB") % (bytes / (1024 * 1024 * 1024))
 
-def pluralize(value, arg='s'):
+    if bytes < 1024:
+        return ungettext("%(size)d byte", "%(size)d bytes", bytes) % {'size': bytes}
+    if bytes < 1024 * 1024:
+        return ugettext("%.1f KB") % (bytes / 1024)
+    if bytes < 1024 * 1024 * 1024:
+        return ugettext("%.1f MB") % (bytes / (1024 * 1024))
+    return ugettext("%.1f GB") % (bytes / (1024 * 1024 * 1024))
+
+def pluralize(value, arg=u's'):
     """
     Returns a plural suffix if the value is not 1, for '1 vote' vs. '2 votes'
     By default, 's' is used as a suffix; if an argument is provided, that string
     is used instead. If the provided argument contains a comma, the text before
     the comma is used for the singular case.
     """
-    if not ',' in arg:
+    if not u',' in arg:
-        arg = ',' + arg
+        arg = u',' + arg
-    bits = arg.split(',')
+    bits = arg.split(u',')
     if len(bits) > 2:
-        return ''
+        return u''
     singular_suffix, plural_suffix = bits[:2]
 
     try:
@@ -562,7 +556,7 @@ def pprint(value):
     try:
         return pformat(value)
     except Exception, e:
-        return "Error in formatting:%s" % e
+        return u"Error in formatting:%s" % force_unicode(e)
 
 # Syntax: register.filter(name of filter, callback)
 register.filter(add)
@@ -582,6 +576,7 @@ register.filter(first)
 register.filter(fix_ampersands)
 register.filter(floatformat)
 register.filter(get_digit)
+register.filter(iriencode)
 register.filter(join)
 register.filter(length)
 register.filter(length_is)

django/template/defaulttags.py

@@ -4,6 +4,7 @@ from django.template import Node, NodeList, Template, Context, resolve_variable
 from django.template import TemplateSyntaxError, VariableDoesNotExist, BLOCK_TAG_START, BLOCK_TAG_END, VARIABLE_TAG_START, VARIABLE_TAG_END, SINGLE_BRACE_START, SINGLE_BRACE_END, COMMENT_TAG_START, COMMENT_TAG_END
 from django.template import get_library, Library, InvalidTemplateLibrary
 from django.conf import settings
+from django.utils.encoding import smart_str, smart_unicode
 from django.utils.itercompat import groupby
 import sys
 import re
@@ -64,8 +65,8 @@ class FirstOfNode(Node):
             except VariableDoesNotExist:
                 continue
             if value:
-                return str(value)
+                return smart_unicode(value)
-        return ''
+        return u''
 
 class ForNode(Node):
     def __init__(self, loopvars, sequence, reversed, nodelist_loop):
@@ -337,7 +338,7 @@ class URLNode(Node):
     def render(self, context):
         from django.core.urlresolvers import reverse, NoReverseMatch
         args = [arg.resolve(context) for arg in self.args]
-        kwargs = dict([(k, v.resolve(context)) for k, v in self.kwargs.items()])
+        kwargs = dict([(smart_str(k,'ascii'), v.resolve(context)) for k, v in self.kwargs.items()])
         try:
             return reverse(self.view_name, args=args, kwargs=kwargs)
         except NoReverseMatch:

django/template/loaders/app_directories.py

@@ -34,7 +34,7 @@ def get_template_sources(template_name, template_dirs=None):
 def load_template_source(template_name, template_dirs=None):
     for filepath in get_template_sources(template_name, template_dirs):
         try:
-            return (open(filepath).read(), filepath)
+            return (open(filepath).read().decode(settings.FILE_CHARSET), filepath)
         except IOError:
             pass
     raise TemplateDoesNotExist, template_name

django/template/loaders/eggs.py

@@ -18,7 +18,7 @@ def load_template_source(template_name, template_dirs=None):
         pkg_name = 'templates/' + template_name
         for app in settings.INSTALLED_APPS:
             try:
-                return (resource_string(app, pkg_name), 'egg:%s:%s ' % (app, pkg_name))
+                return (resource_string(app, pkg_name), 'egg:%s:%s ' % (app, pkg_name)).decode(settings.FILE_CHARSET)
             except:
                 pass
     raise TemplateDoesNotExist, template_name

django/template/loaders/filesystem.py

@@ -14,7 +14,7 @@ def load_template_source(template_name, template_dirs=None):
     tried = []
     for filepath in get_template_sources(template_name, template_dirs):
         try:
-            return (open(filepath).read(), filepath)
+            return (open(filepath).read().decode(settings.FILE_CHARSET), filepath)
         except IOError:
             tried.append(filepath)
     if tried:

django/templatetags/i18n.py

@@ -11,7 +11,7 @@ class GetAvailableLanguagesNode(Node):
 
     def render(self, context):
         from django.conf import settings
-        context[self.variable] = [(k, translation.gettext(v)) for k, v in settings.LANGUAGES]
+        context[self.variable] = [(k, translation.ugettext(v)) for k, v in settings.LANGUAGES]
         return ''
 
 class GetCurrentLanguageNode(Node):
@@ -40,7 +40,7 @@ class TranslateNode(Node):
         if self.noop:
             return value
         else:
-            return translation.gettext(value)
+            return translation.ugettext(value)
 
 class BlockTranslateNode(Node):
     def __init__(self, extra_context, singular, plural=None, countervar=None, counter=None):
@@ -68,9 +68,9 @@ class BlockTranslateNode(Node):
             count = self.counter.resolve(context)
             context[self.countervar] = count
             plural = self.render_token_list(self.plural)
-            result = translation.ngettext(singular, plural, count) % context
+            result = translation.ungettext(singular, plural, count) % context
         else:
-            result = translation.gettext(singular) % context
+            result = translation.ugettext(singular) % context
         context.pop()
         return result
 

django/test/client.py

@@ -10,9 +10,11 @@ from django.core.handlers.base import BaseHandler
 from django.core.handlers.wsgi import WSGIRequest
 from django.core.signals import got_request_exception
 from django.dispatch import dispatcher
-from django.http import urlencode, SimpleCookie, HttpRequest
+from django.http import SimpleCookie, HttpRequest
 from django.test import signals
 from django.utils.functional import curry
+from django.utils.encoding import smart_str
+from django.utils.http import urlencode
 
 BOUNDARY = 'BoUnDaRyStRiNg'
 MULTIPART_CONTENT = 'multipart/form-data; boundary=%s' % BOUNDARY
@@ -61,29 +63,30 @@ def encode_multipart(boundary, data):
     as an application/octet-stream; otherwise, str(value) will be sent.
     """
     lines = []
+    to_str = lambda s: smart_str(s, settings.DEFAULT_CHARSET)
     for (key, value) in data.items():
         if isinstance(value, file):
             lines.extend([
                 '--' + boundary,
-                'Content-Disposition: form-data; name="%s"; filename="%s"' % (key, value.name),
+                'Content-Disposition: form-data; name="%s"; filename="%s"' % (to_str(key), to_str(value.name)),
                 'Content-Type: application/octet-stream',
                 '',
                 value.read()
             ])
-        elif hasattr(value, '__iter__'): 
+        elif hasattr(value, '__iter__'):
             for item in value:
-                lines.extend([ 
+                lines.extend([
-                    '--' + boundary, 
+                    '--' + boundary,
-                    'Content-Disposition: form-data; name="%s"' % key, 
+                    'Content-Disposition: form-data; name="%s"' % to_str(key),
-                    '', 
+                    '',
-                    str(item) 
+                    to_str(item)
                 ])
         else:
             lines.extend([
                 '--' + boundary,
-                'Content-Disposition: form-data; name="%s"' % key,
+                'Content-Disposition: form-data; name="%s"' % to_str(key),
                 '',
-                str(value)
+                to_str(value)
             ])
 
     lines.extend([
@@ -115,7 +118,7 @@ class Client:
         self.defaults = defaults
         self.cookies = SimpleCookie()
         self.exc_info = None
-        
+
     def store_exc_info(self, *args, **kwargs):
         """
         Utility method that can be used to store exceptions when they are
@@ -131,7 +134,7 @@ class Client:
                 return SessionWrapper(cookie.value)
         return {}
     session = property(_session)
-    
+
     def request(self, **request):
         """
         The master request method. Composes the environment dictionary
@@ -179,7 +182,7 @@ class Client:
         # Look for a signalled exception and reraise it
         if self.exc_info:
             raise self.exc_info[1], None, self.exc_info[2]
-        
+
         # Update persistent cookie data
         if response.cookies:
             self.cookies.update(response.cookies)
@@ -243,9 +246,9 @@ class Client:
 
             # Set the session values
             Session.objects.save(obj.session_key, request.session._session,
-                datetime.datetime.now() + datetime.timedelta(seconds=settings.SESSION_COOKIE_AGE))        
+                datetime.datetime.now() + datetime.timedelta(seconds=settings.SESSION_COOKIE_AGE))
 
             return True
         else:
             return False
-            
+