[Android NTP] Move suggestion sections into a separate node.

Also, define how initialization of nodes works, by adding an init()
method that is called after creating a node.

Initialization now happens after the full tree structure has been
created, and recursively processes all nodes in the tree. Subtrees that
are added later use the same pattern; InnerNode now has helper methods
for this.

BUG=616090

Review-Url: https://codereview.chromium.org/2513453004
Cr-Commit-Position: refs/heads/master@{#438135}

chrome/android/java/src/org/chromium/chrome/browser/ntp/cards/AllDismissedItem.java

@@ -14,7 +14,6 @@ import android.widget.TextView;
 
 import org.chromium.chrome.R;
 import org.chromium.chrome.browser.ntp.NewTabPageUma;
-import org.chromium.chrome.browser.ntp.NewTabPageView.NewTabPageManager;
 
 import java.util.Calendar;
 
@@ -48,8 +47,7 @@ public class AllDismissedItem extends OptionalLeaf {
     public static class ViewHolder extends NewTabPageViewHolder {
         private final TextView mBodyTextView;
 
-        public ViewHolder(
-                ViewGroup root, final NewTabPageManager manager, final NewTabPageAdapter adapter) {
+        public ViewHolder(ViewGroup root, final SectionList sections) {
             super(LayoutInflater.from(root.getContext())
                             .inflate(R.layout.new_tab_page_all_dismissed, root, false));
             mBodyTextView = (TextView) itemView.findViewById(R.id.body_text);
@@ -60,9 +58,7 @@ public class AllDismissedItem extends OptionalLeaf {
                         public void onClick(View v) {
                             NewTabPageUma.recordAction(
                                     NewTabPageUma.ACTION_CLICKED_ALL_DISMISSED_REFRESH);
-                            manager.getSuggestionsSource().restoreDismissedCategories();
-                            adapter.resetSections(/*allowEmptySections=*/true);
-                            manager.getSuggestionsSource().fetchRemoteSuggestions();
+                            sections.restoreDismissedSections();
                         }
                     });
         }

chrome/android/java/src/org/chromium/chrome/browser/ntp/cards/ChildNode.java

@@ -16,6 +16,9 @@ public abstract class ChildNode implements TreeNode {
         mParent = parent;
     }
 
+    @Override
+    public void init() {}
+
     protected void notifyItemRangeChanged(int index, int count) {
         mParent.onItemRangeChanged(this, index, count);
     }

chrome/android/java/src/org/chromium/chrome/browser/ntp/cards/InnerNode.java

@@ -108,4 +108,34 @@ public abstract class InnerNode extends ChildNode implements NodeParent {
     public void onItemRangeRemoved(TreeNode child, int index, int count) {
         notifyItemRangeRemoved(getStartingOffsetForChild(child) + index, count);
     }
+
+    @Override
+    public void init() {
+        super.init();
+        for (TreeNode child : getChildren()) {
+            child.init();
+        }
+    }
+
+    /**
+     * Helper method for adding a new child node. Notifies about the inserted items and initializes
+     * the child.
+     *
+     * @param child The child node to be added.
+     */
+    protected void didAddChild(TreeNode child) {
+        int count = child.getItemCount();
+        if (count > 0) onItemRangeInserted(child, 0, count);
+        child.init();
+    }
+
+    /**
+     * Helper method for removing a child node. Notifies about the removed items.
+     *
+     * @param child The child node to be removed.
+     */
+    protected void willRemoveChild(TreeNode child) {
+        int count = child.getItemCount();
+        if (count > 0) onItemRangeRemoved(child, 0, count);
+    }
 }

chrome/android/java/src/org/chromium/chrome/browser/ntp/cards/Leaf.java

@@ -42,6 +42,9 @@ public abstract class Leaf implements TreeNode {
         return 0;
     }
 
+    @Override
+    public void init() {}
+
     /**
      * Display the data for this item.
      * @param holder The view holder that should be updated.

chrome/android/java/src/org/chromium/chrome/browser/ntp/cards/NewTabPageAdapter.java

@@ -17,24 +17,16 @@ import android.view.ViewGroup;
 import org.chromium.base.Log;
 import org.chromium.base.VisibleForTesting;
 import org.chromium.chrome.R;
-import org.chromium.chrome.browser.ntp.NewTabPage.DestructionObserver;
 import org.chromium.chrome.browser.ntp.NewTabPageView.NewTabPageManager;
 import org.chromium.chrome.browser.ntp.UiConfig;
-import org.chromium.chrome.browser.ntp.snippets.CategoryInt;
-import org.chromium.chrome.browser.ntp.snippets.CategoryStatus;
-import org.chromium.chrome.browser.ntp.snippets.CategoryStatus.CategoryStatusEnum;
 import org.chromium.chrome.browser.ntp.snippets.SectionHeaderViewHolder;
 import org.chromium.chrome.browser.ntp.snippets.SnippetArticle;
 import org.chromium.chrome.browser.ntp.snippets.SnippetArticleViewHolder;
-import org.chromium.chrome.browser.ntp.snippets.SnippetsBridge;
-import org.chromium.chrome.browser.ntp.snippets.SnippetsConfig;
 import org.chromium.chrome.browser.ntp.snippets.SuggestionsSource;
 import org.chromium.chrome.browser.offlinepages.OfflinePageBridge;
 
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
+import java.util.Arrays;
 import java.util.List;
-import java.util.Map;
 
 /**
  * A class that handles merging above the fold elements and below the fold cards into an adapter
@@ -42,30 +34,27 @@ import java.util.Map;
  * the above-the-fold view (containing the logo, search box, and most visited tiles) and subsequent
  * elements will be the cards shown to the user
  */
-public class NewTabPageAdapter
-        extends Adapter<NewTabPageViewHolder> implements SuggestionsSource.Observer, NodeParent {
+public class NewTabPageAdapter extends Adapter<NewTabPageViewHolder> implements NodeParent {
     private static final String TAG = "Ntp";
 
     private final NewTabPageManager mNewTabPageManager;
     private final View mAboveTheFoldView;
     private final UiConfig mUiConfig;
     private final ItemTouchCallbacks mItemTouchCallbacks = new ItemTouchCallbacks();
-    private final OfflinePageBridge mOfflinePageBridge;
     private NewTabPageRecyclerView mRecyclerView;
 
     /**
      * List of all child nodes (which can themselves contain multiple child nodes).
      */
-    private final List<TreeNode> mChildren = new ArrayList<>();
+    private final List<TreeNode> mChildren;
+    private final InnerNode mRoot;
+
     private final AboveTheFoldItem mAboveTheFold = new AboveTheFoldItem();
+    private final SectionList mSections;
     private final SignInPromo mSigninPromo;
     private final AllDismissedItem mAllDismissed;
     private final Footer mFooter;
     private final SpacingItem mBottomSpacer = new SpacingItem();
-    private final InnerNode mRoot;
-
-    /** Maps suggestion categories to sections, with stable iteration ordering. */
-    private final Map<Integer, SuggestionsSection> mSections = new LinkedHashMap<>();
 
     private class ItemTouchCallbacks extends ItemTouchHelper.Callback {
         @Override
@@ -143,104 +132,23 @@ public class NewTabPageAdapter
         mNewTabPageManager = manager;
         mAboveTheFoldView = aboveTheFoldView;
         mUiConfig = uiConfig;
-        mOfflinePageBridge = offlinePageBridge;
         mRoot = new InnerNode(this) {
             @Override
             protected List<TreeNode> getChildren() {
                 return mChildren;
             }
-
-            @Override
-            public void onItemRangeChanged(TreeNode child, int index, int count) {
-                if (mChildren.isEmpty()) return; // The sections have not been initialised yet.
-                super.onItemRangeChanged(child, index, count);
-            }
-
-            @Override
-            public void onItemRangeInserted(TreeNode child, int index, int count) {
-                if (mChildren.isEmpty()) return; // The sections have not been initialised yet.
-                super.onItemRangeInserted(child, index, count);
-            }
-
-            @Override
-            public void onItemRangeRemoved(TreeNode child, int index, int count) {
-                if (mChildren.isEmpty()) return; // The sections have not been initialised yet.
-                super.onItemRangeRemoved(child, index, count);
-            }
         };
 
-        mSigninPromo = new SignInPromo(mRoot);
+        mSections = new SectionList(mRoot, mNewTabPageManager, offlinePageBridge);
+        mSigninPromo = new SignInPromo(mRoot, mNewTabPageManager);
         mAllDismissed = new AllDismissedItem(mRoot);
         mFooter = new Footer(mRoot);
-        DestructionObserver signInObserver = mSigninPromo.getObserver();
-        if (signInObserver != null) mNewTabPageManager.addDestructionObserver(signInObserver);
-
-        resetSections(/*alwaysAllowEmptySections=*/false);
-        mNewTabPageManager.getSuggestionsSource().setObserver(this);
-    }
-
-    /**
-     * Resets the sections, reloading the whole new tab page content.
-     * @param alwaysAllowEmptySections Whether sections are always allowed to be displayed when
-     *     they are empty, even when they are normally not.
-     */
-    public void resetSections(boolean alwaysAllowEmptySections) {
-        mSections.clear();
-        mChildren.clear();
-
-        SuggestionsSource suggestionsSource = mNewTabPageManager.getSuggestionsSource();
-        int[] categories = suggestionsSource.getCategories();
-        int[] suggestionsPerCategory = new int[categories.length];
-        int i = 0;
-        for (int category : categories) {
-            int categoryStatus = suggestionsSource.getCategoryStatus(category);
-            if (categoryStatus == CategoryStatus.LOADING_ERROR
-                    || categoryStatus == CategoryStatus.NOT_PROVIDED
-                    || categoryStatus == CategoryStatus.CATEGORY_EXPLICITLY_DISABLED)
-                continue;
-
-            suggestionsPerCategory[i++] =
-                    resetSection(category, categoryStatus, alwaysAllowEmptySections);
-        }
-
-        mNewTabPageManager.trackSnippetsPageImpression(categories, suggestionsPerCategory);
-
-        updateChildren();
-    }
-
-    /**
-     * Resets the section for {@code category}. Removes the section if there are no suggestions for
-     * it and it is not allowed to be empty. Otherwise, creates the section if it is not present
-     * yet. Sets the available suggestions on the section.
-     * @param category The category for which the section must be reset.
-     * @param categoryStatus The category status.
-     * @param alwaysAllowEmptySections Whether sections are always allowed to be displayed when
-     *     they are empty, even when they are normally not.
-     * @return The number of suggestions for the section.
-     */
-    private int resetSection(@CategoryInt int category, @CategoryStatusEnum int categoryStatus,
-            boolean alwaysAllowEmptySections) {
-        SuggestionsSource suggestionsSource = mNewTabPageManager.getSuggestionsSource();
-        List<SnippetArticle> suggestions = suggestionsSource.getSuggestionsForCategory(category);
-        SuggestionsCategoryInfo info = suggestionsSource.getCategoryInfo(category);
-
-        // Do not show an empty section if not allowed.
-        if (suggestions.isEmpty() && !info.showIfEmpty() && !alwaysAllowEmptySections) {
-            mSections.remove(category);
-            return 0;
-        }
-
-        // Create the section if needed.
-        SuggestionsSection section = mSections.get(category);
-        if (section == null) {
-            section = new SuggestionsSection(mRoot, info, mNewTabPageManager, mOfflinePageBridge);
-            mSections.put(category, section);
-        }
 
-        // Add the new suggestions.
-        setSuggestions(category, suggestions, categoryStatus);
+        mChildren = Arrays.asList(
+                mAboveTheFold, mSections, mSigninPromo, mAllDismissed, mFooter, mBottomSpacer);
+        mRoot.init();
 
-        return suggestions.size();
+        updateAllDismissedVisibility();
     }
 
     /** Returns callbacks to configure the interactions with the RecyclerView's items. */
@@ -248,74 +156,6 @@ public class NewTabPageAdapter
         return mItemTouchCallbacks;
     }
 
-    @Override
-    public void onNewSuggestions(@CategoryInt int category) {
-        @CategoryStatusEnum
-        int status = mNewTabPageManager.getSuggestionsSource().getCategoryStatus(category);
-
-        if (!canLoadSuggestions(category, status)) return;
-
-        // We never want to refresh the suggestions if we already have some content.
-        if (mSections.get(category).hasSuggestions()) return;
-
-        List<SnippetArticle> suggestions =
-                mNewTabPageManager.getSuggestionsSource().getSuggestionsForCategory(category);
-
-        Log.d(TAG, "Received %d new suggestions for category %d.", suggestions.size(), category);
-
-        // At first, there might be no suggestions available, we wait until they have been fetched.
-        if (suggestions.isEmpty()) return;
-
-        setSuggestions(category, suggestions, status);
-    }
-
-    @Override
-    public void onMoreSuggestions(@CategoryInt int category, List<SnippetArticle> suggestions) {
-        @CategoryStatusEnum
-        int status = mNewTabPageManager.getSuggestionsSource().getCategoryStatus(category);
-        if (!canLoadSuggestions(category, status)) return;
-
-        setSuggestions(category, suggestions, status);
-    }
-
-    @Override
-    public void onCategoryStatusChanged(@CategoryInt int category, @CategoryStatusEnum int status) {
-        // Observers should not be registered for this state.
-        assert status != CategoryStatus.ALL_SUGGESTIONS_EXPLICITLY_DISABLED;
-
-        // If there is no section for this category there is nothing to do.
-        if (!mSections.containsKey(category)) return;
-
-        switch (status) {
-            case CategoryStatus.NOT_PROVIDED:
-                // The section provider has gone away. Keep open UIs as they are.
-                return;
-
-            case CategoryStatus.CATEGORY_EXPLICITLY_DISABLED:
-            case CategoryStatus.LOADING_ERROR:
-                // Need to remove the entire section from the UI immediately.
-                removeSection(mSections.get(category));
-                return;
-
-            case CategoryStatus.SIGNED_OUT:
-                // TODO(dgn): We currently can only reach this through an old variation parameter.
-            default:
-                mSections.get(category).setStatus(status);
-                return;
-        }
-    }
-
-    @Override
-    public void onSuggestionInvalidated(@CategoryInt int category, String idWithinCategory) {
-        if (!mSections.containsKey(category)) return;
-        mSections.get(category).removeSuggestionById(idWithinCategory);
-    }
-
-    @Override
-    public void onFullRefreshRequired() {
-        resetSections(/*alwaysAllowEmptySections=*/false);
-    }
-
     @Override
     @ItemViewType
     public int getItemViewType(int position) {
@@ -355,7 +195,7 @@ public class NewTabPageAdapter
                 return new Footer.ViewHolder(mRecyclerView, mNewTabPageManager);
 
             case ItemViewType.ALL_DISMISSED:
-                return new AllDismissedItem.ViewHolder(mRecyclerView, mNewTabPageManager, this);
+                return new AllDismissedItem.ViewHolder(mRecyclerView, mSections);
         }
 
         assert false : viewType;
@@ -387,44 +227,12 @@ public class NewTabPageAdapter
         return RecyclerView.NO_POSITION;
     }
 
-    int getBottomSpacerPosition() {
-        return getChildPositionOffset(mBottomSpacer);
-    }
-
     int getLastContentItemPosition() {
         return getChildPositionOffset(hasAllBeenDismissed() ? mAllDismissed : mFooter);
     }
 
-    private void setSuggestions(@CategoryInt int category, List<SnippetArticle> suggestions,
-            @CategoryStatusEnum int status) {
-        // Count the number of suggestions before this category.
-        int globalPositionOffset = 0;
-        for (Map.Entry<Integer, SuggestionsSection> entry : mSections.entrySet()) {
-            if (entry.getKey() == category) break;
-            globalPositionOffset += entry.getValue().getSuggestionsCount();
-        }
-        // Assign global indices to the new suggestions.
-        for (SnippetArticle suggestion : suggestions) {
-            suggestion.mGlobalPosition = globalPositionOffset + suggestion.mPosition;
-        }
-
-        mSections.get(category).addSuggestions(suggestions, status);
-    }
-
-    private void updateChildren() {
-        mChildren.clear();
-        mChildren.add(mAboveTheFold);
-        mChildren.addAll(mSections.values());
-        mChildren.add(mSigninPromo);
-        mChildren.add(mAllDismissed);
-        mChildren.add(mFooter);
-        mChildren.add(mBottomSpacer);
-
-        updateAllDismissedVisibility();
-
-        // TODO(mvanouwerkerk): Notify about the subset of changed items. At least |mAboveTheFold|
-        // has not changed when refreshing from the all dismissed state.
-        notifyDataSetChanged();
+    int getBottomSpacerPosition() {
+        return getChildPositionOffset(mBottomSpacer);
     }
 
     private void updateAllDismissedVisibility() {
@@ -433,17 +241,6 @@ public class NewTabPageAdapter
         mFooter.setVisible(!showAllDismissed);
     }
 
-    private void removeSection(SuggestionsSection section) {
-        mSections.remove(section.getCategory());
-        int startPos = getChildPositionOffset(section);
-        mChildren.remove(section);
-        notifyItemRangeRemoved(startPos, section.getItemCount());
-
-        updateAllDismissedVisibility();
-
-        notifyItemChanged(getBottomSpacerPosition());
-    }
-
     @Override
     public void onItemRangeChanged(TreeNode child, int itemPosition, int itemCount) {
         assert child == mRoot;
@@ -495,7 +292,7 @@ public class NewTabPageAdapter
         switch (itemViewType) {
             case ItemViewType.STATUS:
             case ItemViewType.ACTION:
-                dismissSection(getSuggestionsSection(position));
+                dismissSection(position);
                 return;
 
             case ItemViewType.SNIPPET:
@@ -512,13 +309,10 @@ public class NewTabPageAdapter
         }
     }
 
-    private void dismissSection(SuggestionsSection section) {
-        assert SnippetsConfig.isSectionDismissalEnabled();
-
+    private void dismissSection(int position) {
+        SuggestionsSection section = getSuggestionsSection(position);
+        mSections.dismissSection(section);
         announceItemRemoved(section.getHeaderText());
-
-        mNewTabPageManager.getSuggestionsSource().dismissCategory(section.getCategory());
-        removeSection(section);
     }
 
     private void dismissSuggestion(int position) {
@@ -560,28 +354,15 @@ public class NewTabPageAdapter
         return mSections.isEmpty() && !mSigninPromo.isVisible();
     }
 
-    private boolean canLoadSuggestions(@CategoryInt int category, @CategoryStatusEnum int status) {
-        // We never want to add suggestions from unknown categories.
-        if (!mSections.containsKey(category)) return false;
-
-        // The status may have changed while the suggestions were loading, perhaps they should not
-        // be displayed any more.
-        if (!SnippetsBridge.isCategoryEnabled(status)) {
-            Log.w(TAG, "Received suggestions for a disabled category (id=%d, status=%d)", category,
-                    status);
-            return false;
-        }
-
-        return true;
-    }
-
     /**
      * @param itemPosition The position of an item in the adapter.
      * @return Returns the {@link SuggestionsSection} that contains the item at
      *     {@code itemPosition}, or null if the item is not part of one.
      */
     private SuggestionsSection getSuggestionsSection(int itemPosition) {
-        TreeNode child = mRoot.getChildForPosition(itemPosition);
+        int relativePosition = itemPosition - mRoot.getStartingOffsetForChild(mSections);
+        assert relativePosition >= 0;
+        TreeNode child = mSections.getChildForPosition(relativePosition);
         if (!(child instanceof SuggestionsSection)) return null;
         return (SuggestionsSection) child;
     }
@@ -604,8 +385,8 @@ public class NewTabPageAdapter
         return RecyclerView.NO_POSITION;
     }
 
-    SuggestionsSection getSectionForTesting(@CategoryInt int category) {
-        return mSections.get(category);
+    SectionList getSectionListForTesting() {
+        return mSections;
     }
 
     InnerNode getRootForTesting() {

chrome/android/java/src/org/chromium/chrome/browser/ntp/cards/SectionList.java

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.chromium.chrome.browser.ntp.cards;
+
+import org.chromium.base.Log;
+import org.chromium.chrome.browser.ntp.NewTabPageView.NewTabPageManager;
+import org.chromium.chrome.browser.ntp.snippets.CategoryInt;
+import org.chromium.chrome.browser.ntp.snippets.CategoryStatus;
+import org.chromium.chrome.browser.ntp.snippets.CategoryStatus.CategoryStatusEnum;
+import org.chromium.chrome.browser.ntp.snippets.SnippetArticle;
+import org.chromium.chrome.browser.ntp.snippets.SnippetsBridge;
+import org.chromium.chrome.browser.ntp.snippets.SnippetsConfig;
+import org.chromium.chrome.browser.ntp.snippets.SuggestionsSource;
+import org.chromium.chrome.browser.offlinepages.OfflinePageBridge;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A node in the tree containing a list of all suggestions sections. It listens to changes in the
+ * suggestions source and updates the corresponding sections.
+ */
+public class SectionList extends InnerNode implements SuggestionsSource.Observer {
+    private static final String TAG = "Ntp";
+
+    /** Maps suggestion categories to sections, with stable iteration ordering. */
+    private final Map<Integer, SuggestionsSection> mSections = new LinkedHashMap<>();
+    private final List<TreeNode> mChildren = new ArrayList<>();
+    private final NewTabPageManager mNewTabPageManager;
+    private final OfflinePageBridge mOfflinePageBridge;
+
+    public SectionList(NodeParent parent, NewTabPageManager newTabPageManager,
+            OfflinePageBridge offlinePageBridge) {
+        super(parent);
+        mNewTabPageManager = newTabPageManager;
+        mNewTabPageManager.getSuggestionsSource().setObserver(this);
+        mOfflinePageBridge = offlinePageBridge;
+    }
+
+    @Override
+    public void init() {
+        super.init();
+        resetSections(/* alwaysAllowEmptySections = */ false);
+    }
+
+    @Override
+    protected List<TreeNode> getChildren() {
+        return mChildren;
+    }
+
+    /**
+     * Resets the sections, reloading the whole new tab page content.
+     * @param alwaysAllowEmptySections Whether sections are always allowed to be displayed when
+     *     they are empty, even when they are normally not.
+     */
+    public void resetSections(boolean alwaysAllowEmptySections) {
+        mSections.clear();
+        mChildren.clear();
+
+        SuggestionsSource suggestionsSource = mNewTabPageManager.getSuggestionsSource();
+        int[] categories = suggestionsSource.getCategories();
+        int[] suggestionsPerCategory = new int[categories.length];
+        int i = 0;
+        for (int category : categories) {
+            int categoryStatus = suggestionsSource.getCategoryStatus(category);
+            if (categoryStatus == CategoryStatus.LOADING_ERROR
+                    || categoryStatus == CategoryStatus.NOT_PROVIDED
+                    || categoryStatus == CategoryStatus.CATEGORY_EXPLICITLY_DISABLED)
+                continue;
+
+            suggestionsPerCategory[i++] =
+                    resetSection(category, categoryStatus, alwaysAllowEmptySections);
+        }
+
+        mNewTabPageManager.trackSnippetsPageImpression(categories, suggestionsPerCategory);
+    }
+
+    /**
+     * Resets the section for {@code category}. Removes the section if there are no suggestions for
+     * it and it is not allowed to be empty. Otherwise, creates the section if it is not present
+     * yet. Sets the available suggestions on the section.
+     * @param category The category for which the section must be reset.
+     * @param categoryStatus The category status.
+     * @param alwaysAllowEmptySections Whether sections are always allowed to be displayed when
+     *     they are empty, even when they are normally not.
+     * @return The number of suggestions for the section.
+     */
+    private int resetSection(@CategoryInt int category, @CategoryStatusEnum int categoryStatus,
+            boolean alwaysAllowEmptySections) {
+        SuggestionsSource suggestionsSource = mNewTabPageManager.getSuggestionsSource();
+        List<SnippetArticle> suggestions = suggestionsSource.getSuggestionsForCategory(category);
+        SuggestionsCategoryInfo info = suggestionsSource.getCategoryInfo(category);
+
+        SuggestionsSection section = mSections.get(category);
+
+        // Do not show an empty section if not allowed.
+        if (suggestions.isEmpty() && !info.showIfEmpty() && !alwaysAllowEmptySections) {
+            if (section != null) removeSection(section);
+            return 0;
+        }
+
+        // Create the section if needed.
+        if (section == null) {
+            section = new SuggestionsSection(this, mNewTabPageManager, mOfflinePageBridge, info);
+            mSections.put(category, section);
+            mChildren.add(section);
+            didAddChild(section);
+        }
+
+        // Add the new suggestions.
+        setSuggestions(category, suggestions, categoryStatus);
+
+        return suggestions.size();
+    }
+
+    @Override
+    public void onNewSuggestions(@CategoryInt int category) {
+        @CategoryStatusEnum
+        int status = mNewTabPageManager.getSuggestionsSource().getCategoryStatus(category);
+
+        if (!canLoadSuggestions(category, status)) return;
+
+        // We never want to refresh the suggestions if we already have some content.
+        if (mSections.get(category).hasSuggestions()) return;
+
+        List<SnippetArticle> suggestions =
+                mNewTabPageManager.getSuggestionsSource().getSuggestionsForCategory(category);
+
+        Log.d(TAG, "Received %d new suggestions for category %d.", suggestions.size(), category);
+
+        // At first, there might be no suggestions available, we wait until they have been fetched.
+        if (suggestions.isEmpty()) return;
+
+        setSuggestions(category, suggestions, status);
+    }
+
+    @Override
+    public void onMoreSuggestions(@CategoryInt int category, List<SnippetArticle> suggestions) {
+        @CategoryStatusEnum
+        int status = mNewTabPageManager.getSuggestionsSource().getCategoryStatus(category);
+        if (!canLoadSuggestions(category, status)) return;
+
+        setSuggestions(category, suggestions, status);
+    }
+
+    @Override
+    public void onCategoryStatusChanged(@CategoryInt int category, @CategoryStatusEnum int status) {
+        // Observers should not be registered for this state.
+        assert status != CategoryStatus.ALL_SUGGESTIONS_EXPLICITLY_DISABLED;
+
+        // If there is no section for this category there is nothing to do.
+        if (!mSections.containsKey(category)) return;
+
+        switch (status) {
+            case CategoryStatus.NOT_PROVIDED:
+                // The section provider has gone away. Keep open UIs as they are.
+                return;
+
+            case CategoryStatus.CATEGORY_EXPLICITLY_DISABLED:
+            case CategoryStatus.LOADING_ERROR:
+                // Need to remove the entire section from the UI immediately.
+                removeSection(mSections.get(category));
+                return;
+
+            case CategoryStatus.SIGNED_OUT:
+                resetSection(category, status, /* alwaysAllowEmptySections = */ false);
+                return;
+
+            default:
+                mSections.get(category).setStatus(status);
+                return;
+        }
+    }
+
+    @Override
+    public void onSuggestionInvalidated(@CategoryInt int category, String idWithinCategory) {
+        if (!mSections.containsKey(category)) return;
+        mSections.get(category).removeSuggestionById(idWithinCategory);
+    }
+
+    @Override
+    public void onFullRefreshRequired() {
+        resetSections(/* alwaysAllowEmptySections = */false);
+    }
+
+    private void setSuggestions(@CategoryInt int category, List<SnippetArticle> suggestions,
+            @CategoryStatusEnum int status) {
+        // Count the number of suggestions before this category.
+        int globalPositionOffset = 0;
+        for (Map.Entry<Integer, SuggestionsSection> entry : mSections.entrySet()) {
+            if (entry.getKey() == category) break;
+            globalPositionOffset += entry.getValue().getSuggestionsCount();
+        }
+        // Assign global indices to the new suggestions.
+        for (SnippetArticle suggestion : suggestions) {
+            suggestion.mGlobalPosition = globalPositionOffset + suggestion.mPosition;
+        }
+
+        mSections.get(category).addSuggestions(suggestions, status);
+    }
+
+    private boolean canLoadSuggestions(@CategoryInt int category, @CategoryStatusEnum int status) {
+        // We never want to add suggestions from unknown categories.
+        if (!mSections.containsKey(category)) return false;
+
+        // The status may have changed while the suggestions were loading, perhaps they should not
+        // be displayed any more.
+        if (!SnippetsBridge.isCategoryEnabled(status)) {
+            Log.w(TAG, "Received suggestions for a disabled category (id=%d, status=%d)", category,
+                    status);
+            return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * Dismisses a section.
+     * @param section The section to be dismissed.
+     */
+    public void dismissSection(SuggestionsSection section) {
+        assert SnippetsConfig.isSectionDismissalEnabled();
+
+        mNewTabPageManager.getSuggestionsSource().dismissCategory(section.getCategory());
+        removeSection(section);
+    }
+
+    private void removeSection(SuggestionsSection section) {
+        mSections.remove(section.getCategory());
+        willRemoveChild(section);
+        mChildren.remove(section);
+    }
+
+    /**
+     * Restores any sections that have been dismissed and triggers a new fetch.
+     */
+    public void restoreDismissedSections() {
+        mNewTabPageManager.getSuggestionsSource().restoreDismissedCategories();
+        resetSections(/* allowEmptySections = */ true);
+        mNewTabPageManager.getSuggestionsSource().fetchRemoteSuggestions();
+    }
+
+    /**
+     * @return Whether the list of sections is empty.
+     */
+    public boolean isEmpty() {
+        return mSections.isEmpty();
+    }
+
+    SuggestionsSection getSectionForTesting(@CategoryInt int categoryId) {
+        return mSections.get(categoryId);
+    }
+}

chrome/android/java/src/org/chromium/chrome/browser/ntp/cards/SignInPromo.java

@@ -42,13 +42,24 @@ public class SignInPromo extends OptionalLeaf
     @Nullable
     private final SigninObserver mObserver;
 
-    public SignInPromo(NodeParent parent) {
+    public SignInPromo(NodeParent parent, NewTabPageManager newTabPageManager) {
         super(parent);
         mDismissed = ChromePreferenceManager.getInstance(ContextUtils.getApplicationContext())
                              .getNewTabPageSigninPromoDismissed();
 
-        final SigninManager signinManager = SigninManager.get(ContextUtils.getApplicationContext());
-        mObserver = mDismissed ? null : new SigninObserver(signinManager);
+        SigninManager signinManager = SigninManager.get(ContextUtils.getApplicationContext());
+        if (mDismissed) {
+            mObserver = null;
+        } else {
+            mObserver = new SigninObserver(signinManager);
+            newTabPageManager.addDestructionObserver(mObserver);
+        }
+    }
+
+    @Override
+    public void init() {
+        super.init();
+        SigninManager signinManager = SigninManager.get(ContextUtils.getApplicationContext());
         setVisible(signinManager.isSignInAllowed() && !signinManager.isSignedInOnNative());
     }
 

chrome/android/java/src/org/chromium/chrome/browser/ntp/cards/SuggestionsSection.java

@@ -20,6 +20,7 @@ import org.chromium.chrome.browser.offlinepages.OfflinePageBridge;
 import org.chromium.chrome.browser.offlinepages.OfflinePageItem;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 
@@ -32,19 +33,20 @@ public class SuggestionsSection extends InnerNode {
 
     private final SuggestionsCategoryInfo mCategoryInfo;
     private final OfflinePageBridge mOfflinePageBridge;
-    private final List<TreeNode> mChildren = new ArrayList<>();
 
     // Children
     private final SectionHeader mHeader;
     private final SuggestionsList mSuggestionsList;
     private final StatusItem mStatus;
-    private final ProgressItem mProgressIndicator;
     private final ActionItem mMoreButton;
+    private final ProgressItem mProgressIndicator;
+
+    private final List<TreeNode> mChildren;
 
     private boolean mIsNtpDestroyed = false;
 
-    public SuggestionsSection(NodeParent parent, SuggestionsCategoryInfo info,
-            NewTabPageManager manager, OfflinePageBridge offlinePageBridge) {
+    public SuggestionsSection(NodeParent parent, NewTabPageManager manager,
+            OfflinePageBridge offlinePageBridge, SuggestionsCategoryInfo info) {
         super(parent);
         mCategoryInfo = info;
         mOfflinePageBridge = offlinePageBridge;
@@ -54,11 +56,23 @@ public class SuggestionsSection extends InnerNode {
         mStatus = StatusItem.createNoSuggestionsItem(this);
         mMoreButton = new ActionItem(this);
         mProgressIndicator = new ProgressItem(this);
-        initializeChildren();
+
+        mChildren = Arrays.asList(
+                mHeader,
+                mSuggestionsList,
+                mStatus,
+                mMoreButton,
+                mProgressIndicator);
 
         setupOfflinePageBridgeObserver(manager);
     }
 
+    @Override
+    public void init() {
+        super.init();
+        refreshChildrenVisibility();
+    }
+
     private static class SuggestionsList extends ChildNode implements Iterable<SnippetArticle> {
         private final List<SnippetArticle> mSuggestions = new ArrayList<>();
         private final SuggestionsCategoryInfo mCategoryInfo;
@@ -131,18 +145,6 @@ public class SuggestionsSection extends InnerNode {
         return mChildren;
     }
 
-    private void initializeChildren() {
-        mChildren.add(mHeader);
-        mChildren.add(mSuggestionsList);
-
-        // Optional leaves.
-        mChildren.add(mStatus); // Needs to be refreshed when the status changes.
-
-        mChildren.add(mMoreButton); // Needs to be refreshed when the suggestions change.
-        mChildren.add(mProgressIndicator); // Needs to be refreshed when the suggestions change.
-        refreshChildrenVisibility();
-    }
-
     private void setupOfflinePageBridgeObserver(NewTabPageManager manager) {
         final OfflinePageBridge.OfflinePageModelObserver observer =
                 new OfflinePageBridge.OfflinePageModelObserver() {

chrome/android/java/src/org/chromium/chrome/browser/ntp/cards/TreeNode.java

@@ -4,6 +4,8 @@
 
 package org.chromium.chrome.browser.ntp.cards;
 
+import android.support.annotation.CallSuper;
+
 import org.chromium.chrome.browser.ntp.snippets.SnippetArticle;
 
 /**
@@ -11,6 +13,17 @@ import org.chromium.chrome.browser.ntp.snippets.SnippetArticle;
  */
 interface TreeNode {
     /**
+     * Initialize the node (and any children underneath it). This method should be called after the
+     * node has been added to the tree, i.e. when it is in the list of its parent's children.
+     * The node may notify its parent about changes that happen during initialization.
+     */
+    @CallSuper
+    void init();
+
+    /**
+     * Returns the number of items under this subtree. This method may be called
+     * before initialization.
+     *
      * @return The number of items under this subtree.
      * @see android.support.v7.widget.RecyclerView.Adapter#getItemCount()
      */

chrome/android/java_sources.gni

@@ -600,6 +600,7 @@ chrome_java_sources = [
   "java/src/org/chromium/chrome/browser/ntp/cards/ProgressIndicatorView.java",
   "java/src/org/chromium/chrome/browser/ntp/cards/ProgressItem.java",
   "java/src/org/chromium/chrome/browser/ntp/cards/ProgressViewHolder.java",
+  "java/src/org/chromium/chrome/browser/ntp/cards/SectionList.java",
   "java/src/org/chromium/chrome/browser/ntp/cards/SignInPromo.java",
   "java/src/org/chromium/chrome/browser/ntp/cards/SpacingItem.java",
   "java/src/org/chromium/chrome/browser/ntp/cards/StatusCardViewHolder.java",

chrome/android/junit/src/org/chromium/chrome/browser/ntp/cards/InnerNodeTest.java

@@ -8,6 +8,7 @@ import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assert.assertThat;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
 import org.junit.Before;
@@ -20,7 +21,7 @@ import org.robolectric.annotation.Config;
 import org.chromium.chrome.browser.ntp.snippets.SnippetArticle;
 import org.chromium.testing.local.LocalRobolectricTestRunner;
 
-import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -31,7 +32,7 @@ import java.util.List;
 public class InnerNodeTest {
 
     private static final int[] ITEM_COUNTS = {1, 2, 3, 0, 3, 2, 1};
-    private final List<TreeNode> mChildren = Arrays.asList(new TreeNode[ITEM_COUNTS.length]);
+    private final List<TreeNode> mChildren = new ArrayList<>();
     @Mock private NodeParent mParent;
     private InnerNode mInnerNode;
 
@@ -42,7 +43,7 @@ public class InnerNodeTest {
         for (int i = 0; i < ITEM_COUNTS.length; i++) {
             TreeNode child = mock(TreeNode.class);
             when(child.getItemCount()).thenReturn(ITEM_COUNTS[i]);
-            mChildren.set(i, child);
+            mChildren.add(child);
         }
         mInnerNode = new InnerNode(mParent) {
             @Override
@@ -52,6 +53,14 @@ public class InnerNodeTest {
         };
     }
 
+    @Test
+    public void testInit() {
+        mInnerNode.init();
+        for (TreeNode child : mChildren) {
+            verify(child).init();
+        }
+    }
+
     @Test
     public void testItemCount() {
         assertThat(mInnerNode.getItemCount(), is(12));
@@ -102,6 +111,43 @@ public class InnerNodeTest {
         assertThat(mInnerNode.getSuggestionAt(11), is(article4));
     }
 
+    @Test
+    public void testDidAddChild() {
+        TreeNode child = mock(TreeNode.class);
+        when(child.getItemCount()).thenReturn(23);
+        mChildren.add(3, child);
+        mInnerNode.didAddChild(child);
+
+        // The child should have been initialized and the parent notified about the added items.
+        verify(child).init();
+        verify(mParent).onItemRangeInserted(mInnerNode, 6, 23);
+
+        TreeNode child2 = mock(TreeNode.class);
+        when(child2.getItemCount()).thenReturn(0);
+        mChildren.add(4, child2);
+        mInnerNode.didAddChild(child2);
+        verify(child2).init();
+
+        // The empty child should have been initialized, but there should be no change
+        // notifications.
+        verifyNoMoreInteractions(mParent);
+    }
+
+    @Test
+    public void testWillRemoveChild() {
+        mInnerNode.willRemoveChild(mChildren.get(4));
+        mChildren.remove(4);
+
+        // The parent should have been notified about the removed items.
+        verify(mParent).onItemRangeRemoved(mInnerNode, 6, 3);
+
+        mInnerNode.willRemoveChild(mChildren.get(3));
+        mChildren.remove(3);
+
+        // There should be no change notifications about the empty child.
+        verifyNoMoreInteractions(mParent);
+    }
+
     @Test
     public void testNotifications() {
         mInnerNode.onItemRangeInserted(mChildren.get(0), 0, 23);

chrome/android/junit/src/org/chromium/chrome/browser/ntp/cards/NewTabPageAdapterTest.java

@@ -308,7 +308,8 @@ public class NewTabPageAdapterTest {
     @Test
     @Feature({"Ntp"})
     public void testProgressIndicatorDisplay() {
-        SuggestionsSection section = mAdapter.getSectionForTesting(KnownCategories.ARTICLES);
+        SuggestionsSection section =
+                mAdapter.getSectionListForTesting().getSectionForTesting(KnownCategories.ARTICLES);
         ProgressItem progress = section.getProgressItemForTesting();
 
         mSource.setStatusForCategory(KnownCategories.ARTICLES, CategoryStatus.INITIALIZING);
@@ -402,7 +403,8 @@ public class NewTabPageAdapterTest {
         assertItemsFor(section(3));
 
         // 1.3 - When all suggestions are dismissed
-        SuggestionsSection section42 = mAdapter.getSectionForTesting(category);
+        SuggestionsSection section42 =
+                mAdapter.getSectionListForTesting().getSectionForTesting(category);
         assertSectionMatches(section(3), section42);
         section42.removeSuggestion(articles.get(0));
         section42.removeSuggestion(articles.get(1));
@@ -456,7 +458,8 @@ public class NewTabPageAdapterTest {
         assertItemsFor(section(3).withActionButton());
 
         // 1.3 - When all suggestions are dismissed.
-        SuggestionsSection section42 = mAdapter.getSectionForTesting(category);
+        SuggestionsSection section42 =
+                mAdapter.getSectionListForTesting().getSectionForTesting(category);
         assertSectionMatches(section(3).withActionButton(), section42);
         section42.removeSuggestion(articles.get(0));
         section42.removeSuggestion(articles.get(1));
@@ -480,7 +483,7 @@ public class NewTabPageAdapterTest {
         assertItemsFor(section(3));
 
         // 2.3 - When all suggestions are dismissed.
-        section42 = mAdapter.getSectionForTesting(category);
+        section42 = mAdapter.getSectionListForTesting().getSectionForTesting(category);
         assertSectionMatches(section(3), section42);
         section42.removeSuggestion(articles.get(0));
         section42.removeSuggestion(articles.get(1));
@@ -543,8 +546,6 @@ public class NewTabPageAdapterTest {
     @Test
     @Feature({"Ntp"})
     public void testCategoryOrder() {
-        // Above-the-fold, sign in promo, all-dismissed, footer, spacer.
-        final int basicChildCount = 5;
         FakeSuggestionsSource suggestionsSource = new FakeSuggestionsSource();
         when(mNewTabPageManager.getSuggestionsSource()).thenReturn(suggestionsSource);
         registerCategory(suggestionsSource, KnownCategories.ARTICLES, 0);
@@ -553,17 +554,16 @@ public class NewTabPageAdapterTest {
         registerCategory(suggestionsSource, KnownCategories.DOWNLOADS, 0);
         reloadNtp();
 
-        List<TreeNode> children = mAdapter.getRootForTesting().getChildren();
-        assertEquals(basicChildCount + 4, children.size());
-        assertEquals(AboveTheFoldItem.class, children.get(0).getClass());
+        List<TreeNode> children = mAdapter.getSectionListForTesting().getChildren();
+        assertEquals(4, children.size());
+        assertEquals(SuggestionsSection.class, children.get(0).getClass());
+        assertEquals(KnownCategories.ARTICLES, getCategory(children.get(0)));
         assertEquals(SuggestionsSection.class, children.get(1).getClass());
-        assertEquals(KnownCategories.ARTICLES, getCategory(children.get(1)));
+        assertEquals(KnownCategories.BOOKMARKS, getCategory(children.get(1)));
         assertEquals(SuggestionsSection.class, children.get(2).getClass());
-        assertEquals(KnownCategories.BOOKMARKS, getCategory(children.get(2)));
+        assertEquals(KnownCategories.PHYSICAL_WEB_PAGES, getCategory(children.get(2)));
         assertEquals(SuggestionsSection.class, children.get(3).getClass());
-        assertEquals(KnownCategories.PHYSICAL_WEB_PAGES, getCategory(children.get(3)));
-        assertEquals(SuggestionsSection.class, children.get(4).getClass());
-        assertEquals(KnownCategories.DOWNLOADS, getCategory(children.get(4)));
+        assertEquals(KnownCategories.DOWNLOADS, getCategory(children.get(3)));
 
         // With a different order.
         suggestionsSource = new FakeSuggestionsSource();
@@ -574,17 +574,16 @@ public class NewTabPageAdapterTest {
         registerCategory(suggestionsSource, KnownCategories.BOOKMARKS, 0);
         reloadNtp();
 
-        children = mAdapter.getRootForTesting().getChildren();
-        assertEquals(basicChildCount + 4, children.size());
-        assertEquals(AboveTheFoldItem.class, children.get(0).getClass());
+        children = mAdapter.getSectionListForTesting().getChildren();
+        assertEquals(4, children.size());
+        assertEquals(SuggestionsSection.class, children.get(0).getClass());
+        assertEquals(KnownCategories.ARTICLES, getCategory(children.get(0)));
         assertEquals(SuggestionsSection.class, children.get(1).getClass());
-        assertEquals(KnownCategories.ARTICLES, getCategory(children.get(1)));
+        assertEquals(KnownCategories.PHYSICAL_WEB_PAGES, getCategory(children.get(1)));
         assertEquals(SuggestionsSection.class, children.get(2).getClass());
-        assertEquals(KnownCategories.PHYSICAL_WEB_PAGES, getCategory(children.get(2)));
+        assertEquals(KnownCategories.DOWNLOADS, getCategory(children.get(2)));
         assertEquals(SuggestionsSection.class, children.get(3).getClass());
-        assertEquals(KnownCategories.DOWNLOADS, getCategory(children.get(3)));
-        assertEquals(SuggestionsSection.class, children.get(4).getClass());
-        assertEquals(KnownCategories.BOOKMARKS, getCategory(children.get(4)));
+        assertEquals(KnownCategories.BOOKMARKS, getCategory(children.get(3)));
 
         // With unknown categories.
         suggestionsSource = new FakeSuggestionsSource();
@@ -598,15 +597,14 @@ public class NewTabPageAdapterTest {
         registerCategory(suggestionsSource, 42, 1);
         registerCategory(suggestionsSource, KnownCategories.BOOKMARKS, 1);
 
-        children = mAdapter.getRootForTesting().getChildren();
-        assertEquals(basicChildCount + 3, children.size());
-        assertEquals(AboveTheFoldItem.class, children.get(0).getClass());
+        children = mAdapter.getSectionListForTesting().getChildren();
+        assertEquals(3, children.size());
+        assertEquals(SuggestionsSection.class, children.get(0).getClass());
+        assertEquals(KnownCategories.ARTICLES, getCategory(children.get(0)));
         assertEquals(SuggestionsSection.class, children.get(1).getClass());
-        assertEquals(KnownCategories.ARTICLES, getCategory(children.get(1)));
+        assertEquals(KnownCategories.PHYSICAL_WEB_PAGES, getCategory(children.get(1)));
         assertEquals(SuggestionsSection.class, children.get(2).getClass());
-        assertEquals(KnownCategories.PHYSICAL_WEB_PAGES, getCategory(children.get(2)));
-        assertEquals(SuggestionsSection.class, children.get(3).getClass());
-        assertEquals(KnownCategories.DOWNLOADS, getCategory(children.get(3)));
+        assertEquals(KnownCategories.DOWNLOADS, getCategory(children.get(2)));
     }
 
     @Test
@@ -665,7 +663,7 @@ public class NewTabPageAdapterTest {
         reset(dataObserver);
         suggestionsSource.setSuggestionsForCategory(
                 KnownCategories.ARTICLES, createDummySuggestions(newSuggestionCount));
-        mAdapter.onNewSuggestions(KnownCategories.ARTICLES);
+        mAdapter.getSectionListForTesting().onNewSuggestions(KnownCategories.ARTICLES);
         verify(dataObserver).onItemRangeInserted(2, newSuggestionCount);
         verify(dataObserver).onItemRangeChanged(5 + newSuggestionCount, 1, null); // Spacer refresh
         verify(dataObserver, times(2)).onItemRangeRemoved(2 + newSuggestionCount, 1);
@@ -685,7 +683,8 @@ public class NewTabPageAdapterTest {
         reset(dataObserver);
         suggestionsSource.setSuggestionsForCategory(
                 KnownCategories.ARTICLES, createDummySuggestions(0));
-        mAdapter.onCategoryStatusChanged(KnownCategories.ARTICLES, CategoryStatus.SIGNED_OUT);
+        mAdapter.getSectionListForTesting().onCategoryStatusChanged(
+                KnownCategories.ARTICLES, CategoryStatus.SIGNED_OUT);
         verify(dataObserver).onItemRangeRemoved(2, newSuggestionCount);
         verify(dataObserver).onItemRangeChanged(3, 1, null); // Spacer refresh
         verify(dataObserver).onItemRangeInserted(2, 1); // Status card added
@@ -850,7 +849,7 @@ public class NewTabPageAdapterTest {
 
         // On Sign in, we should reset the sections, bring back suggestions instead of the All
         // Dismissed item.
-        mAdapter.onFullRefreshRequired();
+        mAdapter.getSectionListForTesting().onFullRefreshRequired();
         when(mMockSigninManager.isSignInAllowed()).thenReturn(true);
         signinObserver.onSignedIn();
         // Adapter content:

chrome/android/junit/src/org/chromium/chrome/browser/ntp/cards/SuggestionsSectionTest.java

@@ -347,7 +347,9 @@ public class SuggestionsSectionTest {
     }
 
     private SuggestionsSection createSection(SuggestionsCategoryInfo info) {
-        return new SuggestionsSection(mParent, info, mManager, mBridge);
+        SuggestionsSection section = new SuggestionsSection(mParent, mManager, mBridge, info);
+        section.init();
+        return section;
     }
 
     private OfflinePageItem createOfflinePageItem(String url, long offlineId) {