Split out new module message_store.js.

(imported from commit 57cf3f2b8e74d7c56e3baf75859d5b3646282225)

static/js/compose_fade.js

@@ -21,7 +21,7 @@ exports.set_focused_recipient = function (msg_type) {
         focused_recipient.subject = $('#subject').val();
     } else {
         // Normalize the recipient list so it matches the one used when
-        // adding the message (see add_message_metadata(), zulip.js).
+        // adding the message (see message_store.add_message_metadata()).
         focused_recipient.reply_to = util.normalize_recipients(
                 $('#private_message_recipient').val());
     }

static/js/echo.js

@@ -157,7 +157,7 @@ function insert_local_message(message_request, local_id) {
         });
     }
 
-    insert_new_messages([message]);
+    message_store.insert_new_messages([message]);
     return message.local_id;
 }
 
@@ -182,9 +182,9 @@ exports.try_deliver_locally = function try_deliver_locally(message_request) {
 exports.edit_locally = function edit_locally(message, raw_content, new_topic) {
     message.raw_content = raw_content;
     if (new_topic !== undefined) {
-        process_message_for_recent_subjects(message, true);
+        message_store.process_message_for_recent_subjects(message, true);
         message.subject = new_topic;
-        process_message_for_recent_subjects(message);
+        message_store.process_message_for_recent_subjects(message);
     }
 
     message.content = exports.apply_markdown(raw_content);
@@ -247,7 +247,7 @@ exports.process_from_server = function process_from_server(messages) {
                 }
             }
             locally_processed_ids.push(client_message.id);
-            report_as_received(client_message);
+            message_store.report_as_received(client_message);
             delete waiting_for_ack[client_message.id];
             return false;
         }

static/js/message_store.js

@@ -0,0 +1,588 @@
+var message_store = (function () {
+
+var exports = {};
+exports.msg_metadata_cache = {};
+
+var load_more_enabled = true;
+// If the browser hasn't scrolled away from the top of the page
+// since the last time that we ran load_more_messages(), we do
+// not load_more_messages().
+
+// Returns messages from the given message list in the specified range, inclusive
+exports.message_range = function message_range(msg_list, start, end) {
+    if (start === -1) {
+        blueslip.error("message_range given a start of -1");
+    }
+
+    var all = msg_list.all();
+    var compare = function (a, b) { return a.id < b; };
+
+    var start_idx = util.lower_bound(all, start, compare);
+    var end_idx   = util.lower_bound(all, end,   compare);
+    return all.slice(start_idx, end_idx + 1);
+};
+
+exports.process_message_for_recent_subjects = function process_message_for_recent_subjects(message, remove_message) {
+    var current_timestamp = 0;
+    var count = 0;
+    var stream = message.stream;
+    var canon_subject = stream_data.canonicalized_name(message.subject);
+
+    if (! recent_subjects.has(stream)) {
+        recent_subjects.set(stream, []);
+    } else {
+        recent_subjects.set(stream,
+                            _.filter(recent_subjects.get(stream), function (item) {
+                                var is_duplicate = (item.canon_subject.toLowerCase() === canon_subject.toLowerCase());
+                                if (is_duplicate) {
+                                    current_timestamp = item.timestamp;
+                                    count = item.count;
+                                }
+
+                                return !is_duplicate;
+                            }));
+    }
+
+    var recents = recent_subjects.get(stream);
+
+    if (remove_message !== undefined) {
+        count = count - 1;
+    } else {
+        count = count + 1;
+    }
+
+    if (count !== 0) {
+        recents.push({subject: message.subject,
+                      canon_subject: canon_subject,
+                      count: count,
+                      timestamp: Math.max(message.timestamp, current_timestamp)});
+    }
+
+    recents.sort(function (a, b) {
+        return b.timestamp - a.timestamp;
+    });
+
+    recent_subjects.set(stream, recents);
+};
+
+function set_topic_edit_properties(message) {
+    message.always_visible_topic_edit = false;
+    message.on_hover_topic_edit = false;
+    if (feature_flags.disable_message_editing) {
+        return;
+    }
+
+    // Messages with no topics should always have an edit icon visible
+    // to encourage updating them. Admins can also edit any topic.
+    if (message.subject === compose.empty_subject_placeholder()) {
+        message.always_visible_topic_edit = true;
+    } else if (page_params.is_admin) {
+        message.on_hover_topic_edit = true;
+    }
+}
+
+function add_message_metadata(message) {
+    var cached_msg = exports.msg_metadata_cache[message.id];
+    if (cached_msg !== undefined) {
+        // Copy the match subject and content over if they exist on
+        // the new message
+        if (message.match_subject !== undefined) {
+            cached_msg.match_subject = message.match_subject;
+            cached_msg.match_content = message.match_content;
+        }
+        return cached_msg;
+    }
+
+    var involved_people;
+
+    message.sent_by_me = (message.sender_email === page_params.email);
+
+    message.flags = message.flags || [];
+    message.historical = (message.flags !== undefined &&
+                          message.flags.indexOf('historical') !== -1);
+    message.starred = message.flags.indexOf("starred") !== -1;
+    message.mentioned = message.flags.indexOf("mentioned") !== -1 ||
+                        message.flags.indexOf("wildcard_mentioned") !== -1;
+    message.collapsed = message.flags.indexOf("collapsed") !== -1;
+    message.alerted = message.flags.indexOf("has_alert_word") !== -1;
+    message.is_me_message = message.flags.indexOf("is_me_message") !== -1;
+
+    switch (message.type) {
+    case 'stream':
+        message.is_stream = true;
+        message.stream = message.display_recipient;
+        composebox_typeahead.add_topic(message.stream, message.subject);
+        message.reply_to = message.sender_email;
+
+        exports.process_message_for_recent_subjects(message);
+
+        involved_people = [{'full_name': message.sender_full_name,
+                            'email': message.sender_email}];
+        set_topic_edit_properties(message);
+        break;
+
+    case 'private':
+        message.is_private = true;
+        message.reply_to = util.normalize_recipients(
+                get_private_message_recipient(message, 'email'));
+        message.display_reply_to = get_private_message_recipient(message, 'full_name', 'email');
+
+        involved_people = message.display_recipient;
+        break;
+    }
+
+    // Add new people involved in this message to the people list
+    _.each(involved_people, function (person) {
+        // Do the hasOwnProperty() call via the prototype to avoid problems
+        // with keys like "hasOwnProperty"
+        if (! people.get_by_email(person.email)) {
+            people.add(person);
+        }
+
+        if (people.get_by_email(person.email).full_name !== person.full_name) {
+            people.reify(person);
+        }
+
+        if (message.type === 'private' && message.sent_by_me) {
+            // Track the number of PMs we've sent to this person to improve autocomplete
+            people.get_by_email(person.email).pm_recipient_count += 1;
+        }
+    });
+
+    alert_words.process_message(message);
+    exports.msg_metadata_cache[message.id] = message;
+    return message;
+}
+
+exports.report_as_received = function report_as_received(message) {
+    if (message.sent_by_me) {
+        compose.mark_end_to_end_receive_time(message.id);
+        setTimeout(function () {
+            compose.mark_end_to_end_display_time(message.id);
+        }, 0);
+    }
+};
+
+exports.add_messages = function add_messages(messages, msg_list, opts) {
+    if (!messages) {
+        return;
+    }
+
+    opts = _.extend({messages_are_new: false, delay_render: false}, opts);
+
+    util.destroy_loading_indicator($('#page_loading_indicator'));
+    util.destroy_first_run_message();
+
+    msg_list.add_messages(messages, opts);
+
+    if (msg_list === home_msg_list && opts.messages_are_new) {
+        _.each(messages, function (message) {
+            if (message.local_id === undefined) {
+                exports.report_as_received(message);
+            }
+        });
+    }
+};
+
+function maybe_add_narrowed_messages(messages, msg_list, messages_are_new) {
+    var ids = [];
+    _.each(messages, function (elem) {
+        ids.push(elem.id);
+    });
+
+    channel.post({
+        url:      '/json/messages_in_narrow',
+        idempotent: true,
+        data:     {msg_ids: JSON.stringify(ids),
+                   narrow:  JSON.stringify(narrow.public_operators())},
+        timeout:  5000,
+        success: function (data) {
+            if (msg_list !== current_msg_list) {
+                // We unnarrowed in the mean time
+                return;
+            }
+
+            var new_messages = [];
+            var elsewhere_messages = [];
+            _.each(messages, function (elem) {
+                if (data.messages.hasOwnProperty(elem.id)) {
+                    elem.match_subject = data.messages[elem.id].match_subject;
+                    elem.match_content = data.messages[elem.id].match_content;
+                    new_messages.push(elem);
+                } else {
+                    elsewhere_messages.push(elem);
+                }
+            });
+
+            new_messages = _.map(new_messages, add_message_metadata);
+            exports.add_messages(new_messages, msg_list, {messages_are_new: messages_are_new});
+            unread.process_visible();
+            notifications.possibly_notify_new_messages_outside_viewport(new_messages);
+            notifications.notify_messages_outside_current_search(elsewhere_messages);
+        },
+        error: function (xhr) {
+            // We might want to be more clever here
+            setTimeout(function () {
+                if (msg_list === current_msg_list) {
+                    // Don't actually try again if we unnarrowed
+                    // while waiting
+                    maybe_add_narrowed_messages(messages, msg_list, messages_are_new);
+                }
+            }, 5000);
+        }});
+}
+
+exports.update_messages = function update_messages(events) {
+    _.each(events, function (event) {
+        var msg = all_msg_list.get(event.message_id);
+        if (msg === undefined) {
+            return;
+        }
+
+        msg.alerted = event.flags.indexOf("has_alert_word") !== -1;
+        msg.mentioned = event.flags.indexOf("mentioned") !== -1 ||
+                        event.flags.indexOf("wildcard_mentioned") !== -1;
+
+        ui.un_cache_message_content_height(msg.id);
+
+        if (event.rendered_content !== undefined) {
+            msg.content = event.rendered_content;
+        }
+
+        if (event.subject !== undefined) {
+            // A topic edit may affect multiple messages, listed in
+            // event.message_ids. event.message_id is still the first message
+            // where the user initiated the edit.
+            _.each(event.message_ids, function (id) {
+                var msg = all_msg_list.get(id);
+                if (msg === undefined) {
+                    return;
+                }
+
+                // Remove the recent subjects entry for the old subject;
+                // must be called before we update msg.subject
+                exports.process_message_for_recent_subjects(msg, true);
+                // Update the unread counts; again, this must be called
+                // before we update msg.subject
+                unread.update_unread_subjects(msg, event);
+
+                msg.subject = event.subject;
+                msg.subject_links = event.subject_links;
+                set_topic_edit_properties(msg);
+                // Add the recent subjects entry for the new subject; must
+                // be called after we update msg.subject
+                exports.process_message_for_recent_subjects(msg);
+            });
+        }
+
+        var row = current_msg_list.get_row(event.message_id);
+        if (row.length > 0) {
+            message_edit.end(row);
+        }
+
+        msg.last_edit_timestamp = event.edit_timestamp;
+        delete msg.last_edit_timestr;
+
+        notifications.received_messages([msg]);
+        alert_words.process_message(msg);
+    });
+
+    home_msg_list.rerender();
+    if (current_msg_list === narrowed_msg_list) {
+        narrowed_msg_list.rerender();
+    }
+    unread.update_unread_counts();
+    stream_list.update_streams_sidebar();
+};
+
+exports.insert_new_messages = function insert_new_messages(messages) {
+    messages = _.map(messages, add_message_metadata);
+
+    if (feature_flags.summarize_read_while_narrowed) {
+        _.each(messages, function (message) {
+            if (message.sent_by_me) {
+                summary.maybe_mark_summarized(message);
+            }
+        });
+    }
+
+    // You must add add messages to home_msg_list BEFORE
+    // calling unread.process_loaded_messages.
+    exports.add_messages(messages, home_msg_list, {messages_are_new: true});
+    exports.add_messages(messages, all_msg_list, {messages_are_new: true});
+
+    if (narrow.active()) {
+        if (narrow.filter().can_apply_locally()) {
+            exports.add_messages(messages, narrowed_msg_list, {messages_are_new: true});
+            notifications.possibly_notify_new_messages_outside_viewport(messages);
+        } else {
+            // if we cannot apply locally, we have to wait for this callback to happen to notify
+            maybe_add_narrowed_messages(messages, narrowed_msg_list, true);
+        }
+    } else {
+        notifications.possibly_notify_new_messages_outside_viewport(messages);
+    }
+
+    unread.process_loaded_messages(messages);
+
+    if (narrow.narrowed_by_reply()) {
+        // If you send a message when narrowed to a recipient, move the
+        // pointer to it.
+
+        var i;
+        var selected_id = current_msg_list.selected_id();
+
+        // Iterate backwards to find the last message sent_by_me, stopping at
+        // the pointer position.
+        for (i = messages.length-1; i>=0; i--){
+            var id = messages[i].id;
+            if (id <= selected_id) {
+                break;
+            }
+            if (messages[i].sent_by_me && current_msg_list.get(id) !== undefined) {
+                // If this is a reply we just sent, advance the pointer to it.
+                current_msg_list.select_id(messages[i].id, {then_scroll: true,
+                                                            from_scroll: true});
+                break;
+            }
+        }
+    }
+
+    unread.process_visible();
+    notifications.received_messages(messages);
+    stream_list.update_streams_sidebar();
+};
+
+function process_result(messages, opts) {
+    $('#get_old_messages_error').hide();
+
+    if ((messages.length === 0) && (current_msg_list === narrowed_msg_list) &&
+        narrowed_msg_list.empty()) {
+        // Even after trying to load more messages, we have no
+        // messages to display in this narrow.
+        narrow.show_empty_narrow_message();
+    }
+
+    messages = _.map(messages, add_message_metadata);
+
+    // If we're loading more messages into the home view, save them to
+    // the all_msg_list as well, as the home_msg_list is reconstructed
+    // from all_msg_list.
+    if (opts.msg_list === home_msg_list) {
+        unread.process_loaded_messages(messages);
+        exports.add_messages(messages, all_msg_list, {messages_are_new: false});
+    }
+
+    if (messages.length !== 0 && !opts.cont_will_add_messages) {
+        exports.add_messages(messages, opts.msg_list, {messages_are_new: false});
+    }
+
+    stream_list.update_streams_sidebar();
+
+    if (opts.cont !== undefined) {
+        opts.cont(messages);
+    }
+}
+
+function get_old_messages_success(data, opts) {
+    if (tutorial.is_running()) {
+        // Don't actually process the messages until the tutorial is
+        // finished, but do disable the loading indicator so it isn't
+        // distracting in the background
+        util.destroy_loading_indicator($('#page_loading_indicator'));
+        tutorial.defer(function () { get_old_messages_success(data, opts); });
+        return;
+    }
+
+    if (opts.msg_list.narrowed && opts.msg_list !== current_msg_list) {
+        // We unnarrowed before receiving new messages so
+        // don't bother processing the newly arrived messages.
+        return;
+    }
+    if (! data) {
+        // The server occationally returns no data during a
+        // restart.  Ignore those responses and try again
+        setTimeout(function () {
+            exports.load_old_messages(opts);
+        }, 0);
+        return;
+    }
+
+    process_result(data.messages, opts);
+    ui.resize_bottom_whitespace();
+}
+
+exports.load_old_messages = function load_old_messages(opts) {
+    opts = _.extend({cont_will_add_messages: false}, opts);
+
+    var data = {anchor: opts.anchor,
+                num_before: opts.num_before,
+                num_after: opts.num_after};
+
+    if (opts.msg_list.narrowed && narrow.active()) {
+        var operators = narrow.public_operators();
+        if (page_params.narrow !== undefined) {
+            operators = operators.concat(page_params.narrow);
+        }
+        data.narrow = JSON.stringify(operators);
+    }
+    if (opts.msg_list === home_msg_list && page_params.narrow_stream !== undefined) {
+        data.narrow = JSON.stringify(page_params.narrow);
+    }
+
+    channel.post({
+        url:      '/json/get_old_messages',
+        data:     data,
+        idempotent: true,
+        success: function (data) {
+            get_old_messages_success(data, opts);
+        },
+        error: function (xhr, error_type, exn) {
+            if (opts.msg_list.narrowed && opts.msg_list !== current_msg_list) {
+                // We unnarrowed before getting an error so don't
+                // bother trying again or doing further processing.
+                return;
+            }
+            if (xhr.status === 400) {
+                // Bad request: We probably specified a narrow operator
+                // for a nonexistent stream or something.  We shouldn't
+                // retry or display a connection error.
+                //
+                // FIXME: Warn the user when this has happened?
+                process_result([], opts);
+                return;
+            }
+
+            // We might want to be more clever here
+            $('#get_old_messages_error').show();
+            setTimeout(function () {
+                exports.load_old_messages(opts);
+            }, 5000);
+        }
+    });
+};
+
+exports.reset_load_more_status = function reset_load_more_status() {
+    load_more_enabled = true;
+    have_scrolled_away_from_top = true;
+    ui.hide_loading_more_messages_indicator();
+};
+
+exports.load_more_messages = function load_more_messages(msg_list) {
+    var batch_size = 100;
+    var oldest_message_id;
+    if (!load_more_enabled) {
+        return;
+    }
+    ui.show_loading_more_messages_indicator();
+    load_more_enabled = false;
+    if (msg_list.first() === undefined) {
+        oldest_message_id = page_params.initial_pointer;
+    } else {
+        oldest_message_id = msg_list.first().id;
+    }
+    exports.load_old_messages({
+        anchor: oldest_message_id.toFixed(),
+        num_before: batch_size,
+        num_after: 0,
+        msg_list: msg_list,
+        cont: function (messages) {
+            ui.hide_loading_more_messages_indicator();
+            if (messages.length >= batch_size) {
+                load_more_enabled = true;
+            }
+        }
+    });
+};
+
+$(function () {
+    // get the initial message list
+    function load_more(messages) {
+
+        // If we received the initially selected message, select it on the client side,
+        // but not if the user has already selected another one during load.
+        //
+        // We fall back to the closest selected id, as the user may have removed
+        // a stream from the home before already
+        if (home_msg_list.selected_id() === -1 && !home_msg_list.empty()) {
+            home_msg_list.select_id(page_params.initial_pointer,
+                {then_scroll: true, use_closest: true});
+        }
+
+        // catch the user up
+        if (messages.length !== 0) {
+            var latest_id = messages[messages.length-1].id;
+            if (latest_id < page_params.max_message_id) {
+                exports.load_old_messages({
+                    anchor: latest_id.toFixed(),
+                    num_before: 0,
+                    num_after: 400,
+                    msg_list: home_msg_list,
+                    cont: load_more
+                });
+                return;
+            }
+        }
+
+        server_events.home_view_loaded();
+
+        // backfill more messages after the user is idle
+        var backfill_batch_size = 1000;
+        $(document).idle({'idle': 1000*10,
+                          'onIdle': function () {
+                              var first_id = all_msg_list.first().id;
+                              exports.load_old_messages({
+                                  anchor: first_id,
+                                  num_before: backfill_batch_size,
+                                  num_after: 0,
+                                  msg_list: home_msg_list
+                              });
+                          }});
+    }
+
+    if (page_params.have_initial_messages) {
+        exports.load_old_messages({
+            anchor: page_params.initial_pointer,
+            num_before: 200,
+            num_after: 200,
+            msg_list: home_msg_list,
+            cont: load_more
+        });
+    } else {
+        server_events.home_view_loaded();
+    }
+
+    $(document).on('message_id_changed', function (event) {
+        var old_id = event.old_id, new_id = event.new_id;
+        if (furthest_read === old_id) {
+            furthest_read = new_id;
+        }
+        if (exports.msg_metadata_cache[old_id]) {
+            exports.msg_metadata_cache[new_id] = exports.msg_metadata_cache[old_id];
+            delete exports.msg_metadata_cache[old_id];
+        }
+
+        // This handler cannot be in the MessageList constructor, which is the logical place
+        // If it's there, the event handler creates a closure with a reference to the message
+        // list itself. When narrowing, the old narrow message list is discarded and a new one
+        // created, but due to the closure, the old list is not garbage collected. This also leads
+        // to the old list receiving the change id events, and throwing errors as it does not
+        // have the messages that you would expect in its internal data structures.
+        _.each([all_msg_list, home_msg_list, narrowed_msg_list], function (msg_list) {
+            if (msg_list !== undefined) {
+                msg_list.change_message_id(old_id, new_id);
+
+                if (msg_list.view !== undefined) {
+                    msg_list.view.change_message_id(old_id, new_id);
+                }
+            }
+        });
+    });
+});
+
+return exports;
+
+}());
+if (typeof module !== 'undefined') {
+    module.exports = message_store;
+}

static/js/narrow.js

@@ -252,11 +252,11 @@ exports.activate = function (raw_operators, opts) {
     // the message we want anyway or if the filter can't be applied
     // locally.
     if (all_msg_list.get(then_select_id) !== undefined && current_filter.can_apply_locally()) {
-        add_messages(all_msg_list.all(), narrowed_msg_list, {delay_render: true});
+        message_store.add_messages(all_msg_list.all(), narrowed_msg_list, {delay_render: true});
     }
 
     var defer_selecting_closest = narrowed_msg_list.empty();
-    load_old_messages({
+    message_store.load_old_messages({
         anchor: then_select_id.toFixed(),
         num_before: 50,
         num_after: 50,
@@ -273,7 +273,7 @@ exports.activate = function (raw_operators, opts) {
     });
 
     if (! defer_selecting_closest) {
-        reset_load_more_status();
+        message_store.reset_load_more_status();
         maybe_select_closest();
     } else {
         ui.show_loading_more_messages_indicator();
@@ -317,7 +317,7 @@ exports.by = function (operator, operand, opts) {
 
 exports.by_subject = function (target_id, opts) {
     // don't use current_msg_list as it won't work for muted messages or for out-of-narrow links
-    var original = msg_metadata_cache[target_id];
+    var original = message_store.msg_metadata_cache[target_id];
     if (original.type !== 'stream') {
         // Only stream messages have subjects, but the
         // user wants us to narrow in some way.
@@ -336,7 +336,7 @@ exports.by_subject = function (target_id, opts) {
 exports.by_recipient = function (target_id, opts) {
     opts = _.defaults({}, opts, {then_select_id: target_id});
     // don't use current_msg_list as it won't work for muted messages or for out-of-narrow links
-    var message = msg_metadata_cache[target_id];
+    var message = message_store.msg_metadata_cache[target_id];
     unread.mark_message_as_read(message);
     switch (message.type) {
     case 'private':
@@ -361,7 +361,7 @@ exports.by_id = function (target_id, opts) {
 
 exports.by_conversation_and_time = function (target_id, opts) {
     var args = [["near", target_id]];
-    var original = msg_metadata_cache[target_id];
+    var original = message_store.msg_metadata_cache[target_id];
     opts = _.defaults({}, opts, {then_select_id: target_id});
 
     if (original.type !== 'stream') {
@@ -402,7 +402,7 @@ exports.deactivate = function () {
     ui.condense_and_collapse($("#zhome tr.message_row"));
 
     $('#search_query').val('');
-    reset_load_more_status();
+    message_store.reset_load_more_status();
     hashchange.save_narrow();
 
     if (current_msg_list.selected_id() !== -1) {

static/js/server_events.js

@@ -123,7 +123,7 @@ function get_events_success(events) {
                 break;
             case 'read':
                 var msgs_to_update = _.map(event.messages, function (message_id) {
-                    return msg_metadata_cache[message_id];
+                    return message_store.msg_metadata_cache[message_id];
                 });
                 unread.mark_messages_as_read(msgs_to_update, {from: "server"});
                 break;
@@ -150,7 +150,7 @@ function get_events_success(events) {
 
     if (messages.length !== 0) {
         messages = echo.process_from_server(messages);
-        insert_new_messages(messages);
+        message_store.insert_new_messages(messages);
     }
 
     if (new_pointer !== undefined
@@ -166,7 +166,7 @@ function get_events_success(events) {
     }
 
     if (messages_to_update.length !== 0) {
-        update_messages(messages_to_update);
+        message_store.update_messages(messages_to_update);
     }
 }
 

static/js/subs.js

@@ -82,7 +82,7 @@ function update_in_home_view(sub, value) {
         home_msg_list.clear({clear_selected_id: false});
 
         // Recreate the home_msg_list with the newly filtered all_msg_list
-        add_messages(all_msg_list.all(), home_msg_list);
+        message_store.add_messages(all_msg_list.all(), home_msg_list);
 
         // Ensure we're still at the same scroll position
         if (ui.home_tab_obscured()) {

static/js/ui.js

@@ -1763,7 +1763,7 @@ function scroll_finished() {
         if (viewport.scrollTop() === 0 &&
             have_scrolled_away_from_top) {
             have_scrolled_away_from_top = false;
-            load_more_messages(current_msg_list);
+            message_store.load_more_messages(current_msg_list);
         } else if (!have_scrolled_away_from_top) {
             have_scrolled_away_from_top = true;
         }

static/js/zulip.js

@@ -17,10 +17,6 @@ var recent_subjects = new Dict({fold_case: true});
 var queued_mark_as_read = [];
 var queued_flag_timer;
 
-var load_more_enabled = true;
-// If the browser hasn't scrolled away from the top of the page
-// since the last time that we ran load_more_messages(), we do
-// not load_more_messages().
 var have_scrolled_away_from_top = true;
 
 // Toggles re-centering the pointer in the window
@@ -189,30 +185,6 @@ function get_private_message_recipient(message, attr, fallback_attr) {
     return recipient;
 }
 
-// Returns messages from the given message list in the specified range, inclusive
-function message_range(msg_list, start, end) {
-    if (start === -1) {
-        blueslip.error("message_range given a start of -1");
-    }
-
-    var all = msg_list.all();
-    var compare = function (a, b) { return a.id < b; };
-
-    var start_idx = util.lower_bound(all, start, compare);
-    var end_idx   = util.lower_bound(all, end,   compare);
-    return all.slice(start_idx, end_idx + 1);
-}
-
-// Moving this to unread.js is a little messy because there is already
-// an unread.process_loaded_messages (which this calls)
-function process_loaded_for_unread(messages) {
-    activity.process_loaded_messages(messages);
-    activity.update_huddles();
-    unread.process_loaded_messages(messages);
-    unread.update_unread_counts();
-    ui.resize_page_components();
-}
-
 function respond_to_message(opts) {
     var message, msg_type;
     // Before initiating a reply to a message, if there's an
@@ -295,480 +267,6 @@ function unconditionally_send_pointer_update() {
     }
 }
 
-function process_message_for_recent_subjects(message, remove_message) {
-    var current_timestamp = 0;
-    var count = 0;
-    var stream = message.stream;
-    var canon_subject = stream_data.canonicalized_name(message.subject);
-
-    if (! recent_subjects.has(stream)) {
-        recent_subjects.set(stream, []);
-    } else {
-        recent_subjects.set(stream,
-                            _.filter(recent_subjects.get(stream), function (item) {
-                                var is_duplicate = (item.canon_subject.toLowerCase() === canon_subject.toLowerCase());
-                                if (is_duplicate) {
-                                    current_timestamp = item.timestamp;
-                                    count = item.count;
-                                }
-
-                                return !is_duplicate;
-                            }));
-    }
-
-    var recents = recent_subjects.get(stream);
-
-    if (remove_message !== undefined) {
-        count = count - 1;
-    } else {
-        count = count + 1;
-    }
-
-    if (count !== 0) {
-        recents.push({subject: message.subject,
-                      canon_subject: canon_subject,
-                      count: count,
-                      timestamp: Math.max(message.timestamp, current_timestamp)});
-    }
-
-    recents.sort(function (a, b) {
-        return b.timestamp - a.timestamp;
-    });
-
-    recent_subjects.set(stream, recents);
-}
-
-function set_topic_edit_properties(message) {
-    message.always_visible_topic_edit = false;
-    message.on_hover_topic_edit = false;
-    if (feature_flags.disable_message_editing) {
-        return;
-    }
-
-    // Messages with no topics should always have an edit icon visible
-    // to encourage updating them. Admins can also edit any topic.
-    if (message.subject === compose.empty_subject_placeholder()) {
-        message.always_visible_topic_edit = true;
-    } else if (page_params.is_admin) {
-        message.on_hover_topic_edit = true;
-    }
-}
-
-var msg_metadata_cache = {};
-function add_message_metadata(message) {
-    var cached_msg = msg_metadata_cache[message.id];
-    if (cached_msg !== undefined) {
-        // Copy the match subject and content over if they exist on
-        // the new message
-        if (message.match_subject !== undefined) {
-            cached_msg.match_subject = message.match_subject;
-            cached_msg.match_content = message.match_content;
-        }
-        return cached_msg;
-    }
-
-    var involved_people;
-
-    message.sent_by_me = (message.sender_email === page_params.email);
-
-    message.flags = message.flags || [];
-    message.historical = (message.flags !== undefined &&
-                          message.flags.indexOf('historical') !== -1);
-    message.starred = message.flags.indexOf("starred") !== -1;
-    message.mentioned = message.flags.indexOf("mentioned") !== -1 ||
-                        message.flags.indexOf("wildcard_mentioned") !== -1;
-    message.collapsed = message.flags.indexOf("collapsed") !== -1;
-    message.alerted = message.flags.indexOf("has_alert_word") !== -1;
-    message.is_me_message = message.flags.indexOf("is_me_message") !== -1;
-
-    switch (message.type) {
-    case 'stream':
-        message.is_stream = true;
-        message.stream = message.display_recipient;
-        composebox_typeahead.add_topic(message.stream, message.subject);
-        message.reply_to = message.sender_email;
-
-        process_message_for_recent_subjects(message);
-
-        involved_people = [{'full_name': message.sender_full_name,
-                            'email': message.sender_email}];
-        set_topic_edit_properties(message);
-        break;
-
-    case 'private':
-        message.is_private = true;
-        message.reply_to = util.normalize_recipients(
-                get_private_message_recipient(message, 'email'));
-        message.display_reply_to = get_private_message_recipient(message, 'full_name', 'email');
-
-        involved_people = message.display_recipient;
-        break;
-    }
-
-    // Add new people involved in this message to the people list
-    _.each(involved_people, function (person) {
-        // Do the hasOwnProperty() call via the prototype to avoid problems
-        // with keys like "hasOwnProperty"
-        if (! people.get_by_email(person.email)) {
-            people.add(person);
-        }
-
-        if (people.get_by_email(person.email).full_name !== person.full_name) {
-            people.reify(person);
-        }
-
-        if (message.type === 'private' && message.sent_by_me) {
-            // Track the number of PMs we've sent to this person to improve autocomplete
-            people.get_by_email(person.email).pm_recipient_count += 1;
-        }
-    });
-
-    alert_words.process_message(message);
-    msg_metadata_cache[message.id] = message;
-    return message;
-}
-
-function report_as_received(message) {
-    if (message.sent_by_me) {
-        compose.mark_end_to_end_receive_time(message.id);
-        setTimeout(function () {
-            compose.mark_end_to_end_display_time(message.id);
-        }, 0);
-    }
-}
-
-function add_messages(messages, msg_list, opts) {
-    if (!messages) {
-        return;
-    }
-
-    opts = _.extend({messages_are_new: false, delay_render: false}, opts);
-
-    util.destroy_loading_indicator($('#page_loading_indicator'));
-    util.destroy_first_run_message();
-
-    msg_list.add_messages(messages, opts);
-
-    if (msg_list === home_msg_list && opts.messages_are_new) {
-        _.each(messages, function (message) {
-            if (message.local_id === undefined) {
-                report_as_received(message);
-            }
-        });
-    }
-}
-
-function maybe_add_narrowed_messages(messages, msg_list, messages_are_new) {
-    var ids = [];
-    _.each(messages, function (elem) {
-        ids.push(elem.id);
-    });
-
-    channel.post({
-        url:      '/json/messages_in_narrow',
-        idempotent: true,
-        data:     {msg_ids: JSON.stringify(ids),
-                   narrow:  JSON.stringify(narrow.public_operators())},
-        timeout:  5000,
-        success: function (data) {
-            if (msg_list !== current_msg_list) {
-                // We unnarrowed in the mean time
-                return;
-            }
-
-            var new_messages = [];
-            var elsewhere_messages = [];
-            _.each(messages, function (elem) {
-                if (data.messages.hasOwnProperty(elem.id)) {
-                    elem.match_subject = data.messages[elem.id].match_subject;
-                    elem.match_content = data.messages[elem.id].match_content;
-                    new_messages.push(elem);
-                } else {
-                    elsewhere_messages.push(elem);
-                }
-            });
-
-            new_messages = _.map(new_messages, add_message_metadata);
-            add_messages(new_messages, msg_list, {messages_are_new: messages_are_new});
-            unread.process_visible();
-            notifications.possibly_notify_new_messages_outside_viewport(new_messages);
-            notifications.notify_messages_outside_current_search(elsewhere_messages);
-        },
-        error: function (xhr) {
-            // We might want to be more clever here
-            setTimeout(function () {
-                if (msg_list === current_msg_list) {
-                    // Don't actually try again if we unnarrowed
-                    // while waiting
-                    maybe_add_narrowed_messages(messages, msg_list, messages_are_new);
-                }
-            }, 5000);
-        }});
-}
-
-function update_messages(events) {
-    _.each(events, function (event) {
-        var msg = all_msg_list.get(event.message_id);
-        if (msg === undefined) {
-            return;
-        }
-
-        msg.alerted = event.flags.indexOf("has_alert_word") !== -1;
-        msg.mentioned = event.flags.indexOf("mentioned") !== -1 ||
-                        event.flags.indexOf("wildcard_mentioned") !== -1;
-
-        ui.un_cache_message_content_height(msg.id);
-
-        if (event.rendered_content !== undefined) {
-            msg.content = event.rendered_content;
-        }
-
-        if (event.subject !== undefined) {
-            // A topic edit may affect multiple messages, listed in
-            // event.message_ids. event.message_id is still the first message
-            // where the user initiated the edit.
-            _.each(event.message_ids, function (id) {
-                var msg = all_msg_list.get(id);
-                if (msg === undefined) {
-                    return;
-                }
-
-                // Remove the recent subjects entry for the old subject;
-                // must be called before we update msg.subject
-                process_message_for_recent_subjects(msg, true);
-                // Update the unread counts; again, this must be called
-                // before we update msg.subject
-                unread.update_unread_subjects(msg, event);
-
-                msg.subject = event.subject;
-                msg.subject_links = event.subject_links;
-                set_topic_edit_properties(msg);
-                // Add the recent subjects entry for the new subject; must
-                // be called after we update msg.subject
-                process_message_for_recent_subjects(msg);
-            });
-        }
-
-        var row = current_msg_list.get_row(event.message_id);
-        if (row.length > 0) {
-            message_edit.end(row);
-        }
-
-        msg.last_edit_timestamp = event.edit_timestamp;
-        delete msg.last_edit_timestr;
-
-        notifications.received_messages([msg]);
-        alert_words.process_message(msg);
-    });
-
-    home_msg_list.rerender();
-    if (current_msg_list === narrowed_msg_list) {
-        narrowed_msg_list.rerender();
-    }
-    unread.update_unread_counts();
-    stream_list.update_streams_sidebar();
-}
-
-function insert_new_messages(messages) {
-    messages = _.map(messages, add_message_metadata);
-
-    if (feature_flags.summarize_read_while_narrowed) {
-        _.each(messages, function (message) {
-            if (message.sent_by_me) {
-                summary.maybe_mark_summarized(message);
-            }
-        });
-    }
-
-    // You must add add messages to home_msg_list BEFORE
-    // calling unread.process_loaded_messages.
-    add_messages(messages, home_msg_list, {messages_are_new: true});
-    add_messages(messages, all_msg_list, {messages_are_new: true});
-
-    if (narrow.active()) {
-        if (narrow.filter().can_apply_locally()) {
-            add_messages(messages, narrowed_msg_list, {messages_are_new: true});
-            notifications.possibly_notify_new_messages_outside_viewport(messages);
-        } else {
-            // if we cannot apply locally, we have to wait for this callback to happen to notify
-            maybe_add_narrowed_messages(messages, narrowed_msg_list, true);
-        }
-    } else {
-        notifications.possibly_notify_new_messages_outside_viewport(messages);
-    }
-
-    unread.process_loaded_messages(messages);
-
-    if (narrow.narrowed_by_reply()) {
-        // If you send a message when narrowed to a recipient, move the
-        // pointer to it.
-
-        var i;
-        var selected_id = current_msg_list.selected_id();
-
-        // Iterate backwards to find the last message sent_by_me, stopping at
-        // the pointer position.
-        for (i = messages.length-1; i>=0; i--){
-            var id = messages[i].id;
-            if (id <= selected_id) {
-                break;
-            }
-            if (messages[i].sent_by_me && current_msg_list.get(id) !== undefined) {
-                // If this is a reply we just sent, advance the pointer to it.
-                current_msg_list.select_id(messages[i].id, {then_scroll: true,
-                                                            from_scroll: true});
-                break;
-            }
-        }
-    }
-
-    unread.process_visible();
-    notifications.received_messages(messages);
-    stream_list.update_streams_sidebar();
-}
-
-function process_result(messages, opts) {
-    $('#get_old_messages_error').hide();
-
-    if ((messages.length === 0) && (current_msg_list === narrowed_msg_list) &&
-        narrowed_msg_list.empty()) {
-        // Even after trying to load more messages, we have no
-        // messages to display in this narrow.
-        narrow.show_empty_narrow_message();
-    }
-
-    messages = _.map(messages, add_message_metadata);
-
-    // If we're loading more messages into the home view, save them to
-    // the all_msg_list as well, as the home_msg_list is reconstructed
-    // from all_msg_list.
-    if (opts.msg_list === home_msg_list) {
-        unread.process_loaded_messages(messages);
-        add_messages(messages, all_msg_list, {messages_are_new: false});
-    }
-
-    if (messages.length !== 0 && !opts.cont_will_add_messages) {
-        add_messages(messages, opts.msg_list, {messages_are_new: false});
-    }
-
-    stream_list.update_streams_sidebar();
-
-    if (opts.cont !== undefined) {
-        opts.cont(messages);
-    }
-}
-
-function get_old_messages_success(data, opts) {
-    if (tutorial.is_running()) {
-        // Don't actually process the messages until the tutorial is
-        // finished, but do disable the loading indicator so it isn't
-        // distracting in the background
-        util.destroy_loading_indicator($('#page_loading_indicator'));
-        tutorial.defer(function () { get_old_messages_success(data, opts); });
-        return;
-    }
-
-    if (opts.msg_list.narrowed && opts.msg_list !== current_msg_list) {
-        // We unnarrowed before receiving new messages so
-        // don't bother processing the newly arrived messages.
-        return;
-    }
-    if (! data) {
-        // The server occationally returns no data during a
-        // restart.  Ignore those responses and try again
-        setTimeout(function () {
-            load_old_messages(opts);
-        }, 0);
-        return;
-    }
-
-    process_result(data.messages, opts);
-    ui.resize_bottom_whitespace();
-}
-
-function load_old_messages(opts) {
-    opts = _.extend({cont_will_add_messages: false}, opts);
-
-    var data = {anchor: opts.anchor,
-                num_before: opts.num_before,
-                num_after: opts.num_after};
-
-    if (opts.msg_list.narrowed && narrow.active()) {
-        var operators = narrow.public_operators();
-        if (page_params.narrow !== undefined) {
-            operators = operators.concat(page_params.narrow);
-        }
-        data.narrow = JSON.stringify(operators);
-    }
-    if (opts.msg_list === home_msg_list && page_params.narrow_stream !== undefined) {
-        data.narrow = JSON.stringify(page_params.narrow);
-    }
-
-    channel.post({
-        url:      '/json/get_old_messages',
-        data:     data,
-        idempotent: true,
-        success: function (data) {
-            get_old_messages_success(data, opts);
-        },
-        error: function (xhr, error_type, exn) {
-            if (opts.msg_list.narrowed && opts.msg_list !== current_msg_list) {
-                // We unnarrowed before getting an error so don't
-                // bother trying again or doing further processing.
-                return;
-            }
-            if (xhr.status === 400) {
-                // Bad request: We probably specified a narrow operator
-                // for a nonexistent stream or something.  We shouldn't
-                // retry or display a connection error.
-                //
-                // FIXME: Warn the user when this has happened?
-                process_result([], opts);
-                return;
-            }
-
-            // We might want to be more clever here
-            $('#get_old_messages_error').show();
-            setTimeout(function () {
-                load_old_messages(opts);
-            }, 5000);
-        }
-    });
-}
-
-function reset_load_more_status() {
-    load_more_enabled = true;
-    have_scrolled_away_from_top = true;
-    ui.hide_loading_more_messages_indicator();
-}
-
-function load_more_messages(msg_list) {
-    var batch_size = 100;
-    var oldest_message_id;
-    if (!load_more_enabled) {
-        return;
-    }
-    ui.show_loading_more_messages_indicator();
-    load_more_enabled = false;
-    if (msg_list.first() === undefined) {
-        oldest_message_id = page_params.initial_pointer;
-    } else {
-        oldest_message_id = msg_list.first().id;
-    }
-    load_old_messages({
-        anchor: oldest_message_id.toFixed(),
-        num_before: batch_size,
-        num_after: 0,
-        msg_list: msg_list,
-        cont: function (messages) {
-            ui.hide_loading_more_messages_indicator();
-            if (messages.length >= batch_size) {
-                load_more_enabled = true;
-            }
-        }
-    });
-}
-
 function fast_forward_pointer() {
     channel.post({
         url: '/json/get_profile',
@@ -848,96 +346,13 @@ function main() {
             // Mark messages between old pointer and new pointer as read
             var messages;
             if (event.id < event.previously_selected) {
-                messages = message_range(event.msg_list, event.id, event.previously_selected);
+                messages = message_store.message_range(event.msg_list, event.id, event.previously_selected);
             } else {
-                messages = message_range(event.msg_list, event.previously_selected, event.id);
+                messages = message_store.message_range(event.msg_list, event.previously_selected, event.id);
             }
             unread.mark_messages_as_read(messages, {from: 'pointer'});
         }
     });
-
-    // get the initial message list
-    function load_more(messages) {
-
-        // If we received the initially selected message, select it on the client side,
-        // but not if the user has already selected another one during load.
-        //
-        // We fall back to the closest selected id, as the user may have removed
-        // a stream from the home before already
-        if (home_msg_list.selected_id() === -1 && !home_msg_list.empty()) {
-            home_msg_list.select_id(page_params.initial_pointer,
-                {then_scroll: true, use_closest: true});
-        }
-
-        // catch the user up
-        if (messages.length !== 0) {
-            var latest_id = messages[messages.length-1].id;
-            if (latest_id < page_params.max_message_id) {
-                load_old_messages({
-                    anchor: latest_id.toFixed(),
-                    num_before: 0,
-                    num_after: 400,
-                    msg_list: home_msg_list,
-                    cont: load_more
-                });
-                return;
-            }
-        }
-
-        server_events.home_view_loaded();
-
-        // backfill more messages after the user is idle
-        var backfill_batch_size = 1000;
-        $(document).idle({'idle': 1000*10,
-                          'onIdle': function () {
-                              var first_id = all_msg_list.first().id;
-                              load_old_messages({
-                                  anchor: first_id,
-                                  num_before: backfill_batch_size,
-                                  num_after: 0,
-                                  msg_list: home_msg_list
-                              });
-                          }});
-    }
-
-    if (page_params.have_initial_messages) {
-        load_old_messages({
-            anchor: page_params.initial_pointer,
-            num_before: 200,
-            num_after: 200,
-            msg_list: home_msg_list,
-            cont: load_more
-        });
-    } else {
-        server_events.home_view_loaded();
-    }
-
-    $(document).on('message_id_changed', function (event) {
-        var old_id = event.old_id, new_id = event.new_id;
-        if (furthest_read === old_id) {
-            furthest_read = new_id;
-        }
-        if (msg_metadata_cache[old_id]) {
-            msg_metadata_cache[new_id] = msg_metadata_cache[old_id];
-            delete msg_metadata_cache[old_id];
-        }
-
-        // This handler cannot be in the MessageList constructor, which is the logical place
-        // If it's there, the event handler creates a closure with a reference to the message
-        // list itself. When narrowing, the old narrow message list is discarded and a new one
-        // created, but due to the closure, the old list is not garbage collected. This also leads
-        // to the old list receiving the change id events, and throwing errors as it does not
-        // have the messages that you would expect in its internal data structures.
-        _.each([all_msg_list, home_msg_list, narrowed_msg_list], function (msg_list) {
-            if (msg_list !== undefined) {
-                msg_list.change_message_id(old_id, new_id);
-
-                if (msg_list.view !== undefined) {
-                    msg_list.view.change_message_id(old_id, new_id);
-                }
-            }
-        });
-    });
 }
 
 $(function () {

tools/jslint/check-all.js

@@ -25,7 +25,7 @@ var globals =
     + ' compose compose_fade rows hotkeys narrow reload notifications_bar search subs'
     + ' composebox_typeahead server_events typeahead_helper notifications hashchange'
     + ' invite ui util activity timerender MessageList MessageListView blueslip unread stream_list'
-    + ' message_edit tab_bar emoji popovers navigate people settings'
+    + ' message_edit tab_bar emoji popovers navigate people settings message_store'
     + ' avatar feature_flags search_suggestion referral stream_color Dict'
     + ' Filter summary admin stream_data muting WinChan muting_ui Socket channel'
     + ' message_flags'
@@ -53,22 +53,14 @@ var globals =
 
     // zulip.js
     + ' all_msg_list home_msg_list narrowed_msg_list current_msg_list'
-    + ' add_messages'
     + ' keep_pointer_in_view unread_messages_read_in_narrow'
     + ' respond_to_message recenter_view last_viewport_movement_direction'
     + ' scroll_to_selected get_private_message_recipient'
-    + ' load_old_messages '
     + ' viewport '
-    + ' load_more_messages reset_load_more_status have_scrolled_away_from_top'
+    + ' have_scrolled_away_from_top'
     + ' maybe_scroll_to_selected recenter_pointer_on_display suppress_scroll_pointer_update'
-    + ' message_range message_in_table'
-    + ' message_unread unread_in_current_view'
     + ' fast_forward_pointer recent_subjects unread_subjects'
-    + ' furthest_read server_furthest_read update_messages'
+    + ' furthest_read server_furthest_read'
-    + ' add_message_metadata'
-    + ' msg_metadata_cache'
-    + ' report_as_received'
-    + ' insert_new_messages process_message_for_recent_subjects'
     ;
 
 

zproject/settings.py

@@ -555,6 +555,7 @@ JS_SPECS = {
             'js/alert_words.js',
             'js/alert_words_ui.js',
             'js/people.js',
+            'js/message_store.js',
             'js/server_events.js',
             'js/zulip.js',
             'js/activity.js',