mirror of
https://github.com/django/django.git
synced 2025-04-25 01:34:38 +00:00
953badbea5
backwards compatible for all practical purposes. Fixed #2391, #2489, #2996, #3322, #3344, #3370, #3406, #3432, #3454, #3492, #3582, #3690, #3878, #3891, #3937, #4039, #4141, #4227, #4286, #4291, #4300, #4452, #4702 git-svn-id: http://code.djangoproject.com/svn/django/trunk@5609 bcc190cf-cafb-0310-a4f2-bffc1f526a37
43 lines
593 B
Python
43 lines
593 B
Python
# Performance note: I benchmarked this code using a set instead of
|
|
# a list for the stopwords and was surprised to find that the list
|
|
# performed /better/ than the set - maybe because it's only a small
|
|
# list.
|
|
|
|
stopwords = '''
|
|
i
|
|
a
|
|
an
|
|
are
|
|
as
|
|
at
|
|
be
|
|
by
|
|
for
|
|
from
|
|
how
|
|
in
|
|
is
|
|
it
|
|
of
|
|
on
|
|
or
|
|
that
|
|
the
|
|
this
|
|
to
|
|
was
|
|
what
|
|
when
|
|
where
|
|
'''.split()
|
|
|
|
def strip_stopwords(sentence):
|
|
"Removes stopwords - also normalizes whitespace"
|
|
words = sentence.split()
|
|
sentence = []
|
|
for word in words:
|
|
if word.lower() not in stopwords:
|
|
sentence.append(word)
|
|
return u' '.join(sentence)
|
|
|