diff --git a/zephyr/migrations/0024_create_escape_html_function.py b/zephyr/migrations/0024_create_escape_html_function.py new file mode 100644 index 0000000000..cdecdf19dc --- /dev/null +++ b/zephyr/migrations/0024_create_escape_html_function.py @@ -0,0 +1,151 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models +from django.conf import settings + +class Migration(SchemaMigration): + + def forwards(self, orm): + if "postgres" not in settings.DATABASES["default"]["ENGINE"]: + return + + # This is a translation of django.util.html.escape + db.execute("""CREATE FUNCTION escape_html(text) RETURNS text IMMUTABLE + LANGUAGE 'sql' AS $$ SELECT replace(replace(replace( + replace(replace($1, '&', '&'), '<', '<'), '>', + '>'), '"', '"'), '''', '''); $$""") + + def backwards(self, orm): + if "postgres" not in settings.DATABASES["default"]["ENGINE"]: + return + + db.execute("DROP FUNCTION escape_html(text)") + + models = { + u'zephyr.client': { + 'Meta': {'object_name': 'Client'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30', 'db_index': 'True'}) + }, + u'zephyr.defaultstream': { + 'Meta': {'unique_together': "(('realm', 'stream'),)", 'object_name': 'DefaultStream'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'realm': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zephyr.Realm']"}), + 'stream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zephyr.Stream']"}) + }, + u'zephyr.huddle': { + 'Meta': {'object_name': 'Huddle'}, + 'huddle_hash': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + u'zephyr.message': { + 'Meta': {'object_name': 'Message'}, + 'content': ('django.db.models.fields.TextField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'pub_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'recipient': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zephyr.Recipient']"}), + 'rendered_content': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'rendered_content_version': ('django.db.models.fields.IntegerField', [], {'null': 'True'}), + 'sender': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zephyr.UserProfile']"}), + 'sending_client': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zephyr.Client']"}), + 'subject': ('django.db.models.fields.CharField', [], {'max_length': '60', 'db_index': 'True'}) + }, + u'zephyr.mituser': { + 'Meta': {'object_name': 'MitUser'}, + 'email': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'status': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + u'zephyr.preregistrationuser': { + 'Meta': {'object_name': 'PreregistrationUser'}, + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'invited_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'referred_by': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zephyr.UserProfile']", 'null': 'True'}), + 'status': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'streams': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['zephyr.Stream']", 'null': 'True', 'symmetrical': 'False'}) + }, + u'zephyr.realm': { + 'Meta': {'object_name': 'Realm'}, + 'domain': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'restricted_to_domain': ('django.db.models.fields.BooleanField', [], {'default': 'True'}) + }, + u'zephyr.recipient': { + 'Meta': {'unique_together': "(('type', 'type_id'),)", 'object_name': 'Recipient'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'type': ('django.db.models.fields.PositiveSmallIntegerField', [], {'db_index': 'True'}), + 'type_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}) + }, + u'zephyr.stream': { + 'Meta': {'unique_together': "(('name', 'realm'),)", 'object_name': 'Stream'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'invite_only': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'db_index': 'True'}), + 'realm': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zephyr.Realm']"}) + }, + u'zephyr.streamcolor': { + 'Meta': {'object_name': 'StreamColor'}, + 'color': ('django.db.models.fields.CharField', [], {'max_length': '10'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'subscription': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zephyr.Subscription']"}) + }, + u'zephyr.subscription': { + 'Meta': {'unique_together': "(('user_profile', 'recipient'),)", 'object_name': 'Subscription'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'color': ('django.db.models.fields.CharField', [], {'default': "'#c2c2c2'", 'max_length': '10'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'in_home_view': ('django.db.models.fields.NullBooleanField', [], {'default': 'True', 'null': 'True', 'blank': 'True'}), + 'notifications': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'recipient': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zephyr.Recipient']"}), + 'user_profile': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zephyr.UserProfile']"}) + }, + u'zephyr.useractivity': { + 'Meta': {'unique_together': "(('user_profile', 'client', 'query'),)", 'object_name': 'UserActivity'}, + 'client': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zephyr.Client']"}), + 'count': ('django.db.models.fields.IntegerField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_visit': ('django.db.models.fields.DateTimeField', [], {}), + 'query': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'user_profile': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zephyr.UserProfile']"}) + }, + u'zephyr.usermessage': { + 'Meta': {'unique_together': "(('user_profile', 'message'),)", 'object_name': 'UserMessage'}, + 'archived': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'flags': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zephyr.Message']"}), + 'user_profile': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zephyr.UserProfile']"}) + }, + u'zephyr.userpresence': { + 'Meta': {'unique_together': "(('user_profile', 'client'),)", 'object_name': 'UserPresence'}, + 'client': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zephyr.Client']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'status': ('django.db.models.fields.PositiveSmallIntegerField', [], {'default': '1'}), + 'timestamp': ('django.db.models.fields.DateTimeField', [], {}), + 'user_profile': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zephyr.UserProfile']"}) + }, + u'zephyr.userprofile': { + 'Meta': {'object_name': 'UserProfile'}, + 'api_key': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75', 'db_index': 'True'}), + 'enable_desktop_notifications': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'enter_sends': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}), + 'full_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_pointer_updater': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'pointer': ('django.db.models.fields.IntegerField', [], {}), + 'realm': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zephyr.Realm']"}), + 'short_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'tutorial_status': ('django.db.models.fields.CharField', [], {'default': "'W'", 'max_length': '1'}) + } + } + + complete_apps = ['zephyr'] diff --git a/zephyr/views.py b/zephyr/views.py index 7af0b1d406..7716536100 100644 --- a/zephyr/views.py +++ b/zephyr/views.py @@ -673,8 +673,20 @@ class NarrowBuilder(object): def do_search(self, query, operand): if "postgres" in settings.DATABASES["default"]["ENGINE"]: - sql = "search_tsvector @@ plainto_tsquery('humbug.english_us_search', %s)" - return query.extra(where=[sql], params=[operand]) + tsquery = "plainto_tsquery('humbug.english_us_search', %s)" + where = "search_tsvector @@ " + tsquery + match_content = "ts_headline('humbug.english_us_search', rendered_content, " \ + + tsquery + ", 'StartSel=\"\", StopSel=, " \ + "HighlightAll=TRUE')" + # We HTML-escape the subject in Postgres to avoid doing a server round-trip + match_subject = "ts_headline('humbug.english_us_search', escape_html(subject), " \ + + tsquery + ", 'StartSel=\"\", StopSel=, " \ + "HighlightAll=TRUE')" + + return query.extra(select={'match_content': match_content, + 'match_subject': match_subject}, + where=[where], + select_params=[operand, operand], params=[operand]) else: for word in operand.split(): query = query.filter(self.pQ(content__icontains=word) | @@ -735,11 +747,14 @@ def get_old_messages_backend(request, user_profile, .order_by('message') num_extra_messages = 1 + is_search = False if narrow is not None: num_extra_messages = 0 build = NarrowBuilder(user_profile, prefix) for operator, operand in narrow: + if operator == 'search': + is_search = True query = build(query, operator, operand) def add_prefix(**kwargs): @@ -767,15 +782,26 @@ def get_old_messages_backend(request, user_profile, # rendered message dict before returning it. We attempt to # bulk-fetch rendered message dicts from memcached using the # 'messages' list. + search_fields = dict() + messages = [] if include_history: user_messages = dict((user_message.message_id, user_message) for user_message in UserMessage.objects.filter(user_profile=user_profile, message__in=query_result)) - messages = query_result + for message in query_result: + messages.append(message) + if is_search: + search_fields[message.id] = dict([('match_subject', message.match_subject), + ('match_content', message.match_content)]) else: user_messages = dict((user_message.message_id, user_message) for user_message in query_result) - messages = [user_message.message for user_message in query_result] + for user_message in query_result: + messages.append(user_message.message) + if is_search: + search_fields[user_message.message.id] = \ + dict([('match_subject', user_message.match_subject), + ('match_content', user_message.match_content)]) bulk_messages = cache_get_many([to_dict_cache_key(message, apply_markdown) for message in messages]) @@ -796,7 +822,10 @@ def get_old_messages_backend(request, user_profile, if message.id in user_messages: flags_dict = user_messages[message.id].flags_dict() elt = bulk_messages.get(to_dict_cache_key(message, apply_markdown))[0] - message_list.append(dict(elt, **flags_dict)) + msg_dict = dict(elt) + msg_dict.update(flags_dict) + msg_dict.update(search_fields.get(elt['id'], {})) + message_list.append(msg_dict) statsd.incr('loaded_old_messages', len(message_list)) ret = {'messages': message_list,