improve search

1. separate logged-in and anonymous endpoints
2. migrate to retrofit + gson, retire SuggestionsFetcher
3. prefixing search with @ or # will only return users or hashtags, respectively
4. add subtitles for locations (address) and hashtags (rough post count)

app/src/main/java/awais/instagrabber/activities/MainActivity.java

@@ -11,7 +11,6 @@ import android.content.ServiceConnection;
 import android.content.res.TypedArray;
 import android.database.MatrixCursor;
 import android.net.Uri;
-import android.os.AsyncTask;
 import android.os.Build;
 import android.os.Bundle;
 import android.os.Handler;
@@ -57,21 +56,22 @@ import java.util.Deque;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import awais.instagrabber.R;
 import awais.instagrabber.adapters.SuggestionsAdapter;
 import awais.instagrabber.asyncs.PostFetcher;
-import awais.instagrabber.asyncs.SuggestionsFetcher;
 import awais.instagrabber.customviews.emoji.EmojiVariantManager;
 import awais.instagrabber.databinding.ActivityMainBinding;
 import awais.instagrabber.fragments.PostViewV2Fragment;
 import awais.instagrabber.fragments.directmessages.DirectMessageInboxFragmentDirections;
 import awais.instagrabber.fragments.main.FeedFragment;
 import awais.instagrabber.fragments.settings.PreferenceKeys;
-import awais.instagrabber.interfaces.FetchListener;
 import awais.instagrabber.models.IntentModel;
 import awais.instagrabber.models.SuggestionModel;
 import awais.instagrabber.models.enums.SuggestionType;
+import awais.instagrabber.repositories.responses.search.SearchItem;
+import awais.instagrabber.repositories.responses.search.SearchResponse;
 import awais.instagrabber.services.ActivityCheckerService;
 import awais.instagrabber.services.DMSyncAlarmReceiver;
 import awais.instagrabber.utils.AppExecutors;
@@ -83,6 +83,10 @@ import awais.instagrabber.utils.TextUtils;
 import awais.instagrabber.utils.Utils;
 import awais.instagrabber.utils.emoji.EmojiParser;
 import awais.instagrabber.viewmodels.AppStateViewModel;
+import awais.instagrabber.webservices.SearchService;
+import retrofit2.Call;
+import retrofit2.Callback;
+import retrofit2.Response;
 
 import static awais.instagrabber.utils.NavigationExtensions.setupWithNavController;
 import static awais.instagrabber.utils.Utils.settingsHelper;
@@ -105,6 +109,7 @@ public class MainActivity extends BaseLanguageActivity implements FragmentManage
     private SuggestionsAdapter suggestionAdapter;
     private AutoCompleteTextView searchAutoComplete;
     private SearchView searchView;
+    private SearchService searchService;
     private boolean showSearch = true;
     private Handler suggestionsFetchHandler;
     private int firstFragmentGraphIndex;
@@ -175,6 +180,7 @@ public class MainActivity extends BaseLanguageActivity implements FragmentManage
             EmojiVariantManager.getInstance();
         });
         initEmojiCompat();
+        searchService = SearchService.getInstance();
         // initDmService();
     }
 
@@ -338,51 +344,84 @@ public class MainActivity extends BaseLanguageActivity implements FragmentManage
         searchView.setOnQueryTextListener(new SearchView.OnQueryTextListener() {
             private boolean searchUser;
             private boolean searchHash;
-            private AsyncTask<?, ?, ?> prevSuggestionAsync;
+            private Call<SearchResponse> prevSuggestionAsync;
             private final String[] COLUMNS = {
                     BaseColumns._ID,
                     Constants.EXTRAS_USERNAME,
                     Constants.EXTRAS_NAME,
                     Constants.EXTRAS_TYPE,
+                    "query",
                     "pfp",
                     "verified"
             };
             private String currentSearchQuery;
 
-            private final FetchListener<SuggestionModel[]> fetchListener = new FetchListener<SuggestionModel[]>() {
+            private final Callback<SearchResponse> cb = new Callback<SearchResponse>() {
                 @Override
-                public void doBefore() {
+                public void onResponse(@NonNull final Call<SearchResponse> call,
-                    suggestionAdapter.changeCursor(null);
+                                       @NonNull final Response<SearchResponse> response) {
-                }
-
-                @Override
-                public void onResult(final SuggestionModel[] result) {
                     final MatrixCursor cursor;
-                    if (result == null) cursor = null;
+                    final SearchResponse body = response.body();
+                    if (body == null) {
+                        cursor = null;
+                        return;
+                    }
+                    final List<SearchItem> result = new ArrayList<SearchItem>();
+                    if (isLoggedIn) {
+                        if (body.getList() != null) result.addAll(searchHash ? body.getList()
+                                .stream()
+                                .filter(i -> i.getUser() == null)
+                                .collect(Collectors.toList()) : body.getList());
+                    }
                     else {
+                        if (body.getUsers() != null && !searchHash) result.addAll(body.getUsers());
+                        if (body.getHashtags() != null) result.addAll(body.getHashtags());
+                        if (body.getPlaces() != null) result.addAll(body.getPlaces());
+                    }
                     cursor = new MatrixCursor(COLUMNS, 0);
-                        for (int i = 0; i < result.length; i++) {
+                    for (int i = 0; i < result.size(); i++) {
-                            final SuggestionModel suggestionModel = result[i];
+                        final SearchItem suggestionModel = result.get(i);
                         if (suggestionModel != null) {
-                                final SuggestionType suggestionType = suggestionModel.getSuggestionType();
+                            Object[] objects = null;
-                                final Object[] objects = {
+                            if (suggestionModel.getUser() != null)
-                                        i,
+                                objects = new Object[]{
-                                        suggestionType == SuggestionType.TYPE_LOCATION ? suggestionModel.getName() : suggestionModel.getUsername(),
+                                        suggestionModel.getPosition(),
-                                        suggestionType == SuggestionType.TYPE_LOCATION ? suggestionModel.getUsername() : suggestionModel.getName(),
+                                        suggestionModel.getUser().getUsername(),
-                                        suggestionType,
+                                        suggestionModel.getUser().getFullName(),
-                                        suggestionModel.getProfilePic(),
+                                        SuggestionType.TYPE_USER,
-                                        suggestionModel.isVerified()};
+                                        suggestionModel.getUser().getUsername(),
-                                if (!searchHash && !searchUser) cursor.addRow(objects);
+                                        suggestionModel.getUser().getProfilePicUrl(),
-                                else {
+                                        suggestionModel.getUser().isVerified()};
-                                    final boolean isCurrHash = suggestionType == SuggestionType.TYPE_HASHTAG;
+                            else if (suggestionModel.getHashtag() != null)
-                                    if (searchHash && isCurrHash || !searchHash && !isCurrHash)
+                                objects = new Object[]{
+                                        suggestionModel.getPosition(),
+                                        suggestionModel.getHashtag().getName(),
+                                        suggestionModel.getHashtag().getSubtitle(),
+                                        SuggestionType.TYPE_HASHTAG,
+                                        suggestionModel.getHashtag().getName(),
+                                        "res:/" + R.drawable.ic_hashtag,
+                                        false};
+                            else if (suggestionModel.getPlace() != null)
+                                objects = new Object[]{
+                                        suggestionModel.getPosition(),
+                                        suggestionModel.getPlace().getTitle(),
+                                        suggestionModel.getPlace().getSubtitle(),
+                                        SuggestionType.TYPE_LOCATION,
+                                        suggestionModel.getPlace().getLocation().getPk(),
+                                        "res:/" + R.drawable.ic_location,
+                                        false};
                             cursor.addRow(objects);
                         }
                     }
-                        }
-                    }
                     suggestionAdapter.changeCursor(cursor);
                 }
+
+                @Override
+                public void onFailure(@NonNull final Call<SearchResponse> call,
+                                      Throwable t) {
+                    if (!call.isCanceled() && t != null)
+                        Log.e(TAG, "Exception on search:", t);
+                }
             };
 
             private final Runnable runnable = () -> {
@@ -401,17 +440,19 @@ public class MainActivity extends BaseLanguageActivity implements FragmentManage
                     if (searchAutoComplete != null) {
                         searchAutoComplete.setThreshold(1);
                     }
-                    prevSuggestionAsync = new SuggestionsFetcher(fetchListener).executeOnExecutor(
+                    prevSuggestionAsync = searchService.search(isLoggedIn,
-                            AsyncTask.THREAD_POOL_EXECUTOR,
                                                                searchUser || searchHash ? currentSearchQuery.substring(1)
-                                                     : currentSearchQuery);
+                                                                    : currentSearchQuery,
+                                                               searchUser ? "user" : (searchHash ? "hashtag" : "blended"));
+                    suggestionAdapter.changeCursor(null);
+                    prevSuggestionAsync.enqueue(cb);
                 }
             };
 
             private void cancelSuggestionsAsync() {
                 if (prevSuggestionAsync != null)
                     try {
-                        prevSuggestionAsync.cancel(true);
+                        prevSuggestionAsync.cancel();
                     } catch (final Exception ignored) {}
             }
 

app/src/main/java/awais/instagrabber/adapters/SuggestionsAdapter.java

@@ -35,12 +35,12 @@ public final class SuggestionsAdapter extends CursorAdapter {
 
     @Override
     public void bindView(@NonNull final View view, final Context context, @NonNull final Cursor cursor) {
-        // i, username, fullname, type, picUrl, verified
+        // i, username, fullname, type, query, picUrl, verified
-        // 0, 1       , 2       , 3   , 4     , 5
+        // 0, 1       , 2       , 3   , 4    , 5     , 6
         final String fullName = cursor.getString(2);
         String username = cursor.getString(1);
-        String picUrl = cursor.getString(4);
+        String picUrl = cursor.getString(5);
-        final boolean verified = cursor.getString(5).charAt(0) == 't';
+        final boolean verified = cursor.getString(6).charAt(0) == 't';
 
         final String type = cursor.getString(3);
         SuggestionType suggestionType = null;
@@ -50,22 +50,14 @@ public final class SuggestionsAdapter extends CursorAdapter {
             Log.e(TAG, "Unknown suggestion type: " + type, e);
         }
         if (suggestionType == null) return;
-        final String query;
+        String query = cursor.getString(4);
         switch (suggestionType) {
             case TYPE_USER:
                 username = '@' + username;
-                query = username;
                 break;
             case TYPE_HASHTAG:
                 username = '#' + username;
-                query = username;
                 break;
-            case TYPE_LOCATION:
-                query = fullName;
-                picUrl = "res:/" + R.drawable.ic_location;
-                break;
-            default:
-                return; // will never come here
         }
 
         if (onSuggestionClickListener != null) {
@@ -75,12 +67,8 @@ public final class SuggestionsAdapter extends CursorAdapter {
         final ItemSuggestionBinding binding = ItemSuggestionBinding.bind(view);
         binding.isVerified.setVisibility(verified ? View.VISIBLE : View.GONE);
         binding.tvUsername.setText(username);
-        if (suggestionType.equals(SuggestionType.TYPE_LOCATION)) {
-            binding.tvFullName.setVisibility(View.GONE);
-        } else {
         binding.tvFullName.setVisibility(View.VISIBLE);
         binding.tvFullName.setText(fullName);
-        }
         binding.ivProfilePic.setImageURI(picUrl);
     }
 

app/src/main/java/awais/instagrabber/asyncs/SuggestionsFetcher.java

@@ -1,113 +0,0 @@
-package awais.instagrabber.asyncs;
-
-import android.os.AsyncTask;
-import android.util.Log;
-
-import org.json.JSONArray;
-import org.json.JSONObject;
-
-import java.io.InterruptedIOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collections;
-
-import awais.instagrabber.BuildConfig;
-import awais.instagrabber.interfaces.FetchListener;
-import awais.instagrabber.models.SuggestionModel;
-import awais.instagrabber.models.enums.SuggestionType;
-import awais.instagrabber.utils.Constants;
-import awais.instagrabber.utils.NetworkUtils;
-import awais.instagrabber.utils.UrlEncoder;
-
-public final class SuggestionsFetcher extends AsyncTask<String, String, SuggestionModel[]> {
-    private final FetchListener<SuggestionModel[]> fetchListener;
-
-    public SuggestionsFetcher(final FetchListener<SuggestionModel[]> fetchListener) {
-        this.fetchListener = fetchListener;
-    }
-
-    @Override
-    protected void onPreExecute() {
-        if (fetchListener != null) fetchListener.doBefore();
-    }
-
-    @Override
-    protected SuggestionModel[] doInBackground(final String... params) {
-        SuggestionModel[] result = null;
-        try {
-            final HttpURLConnection conn = (HttpURLConnection) new URL("https://www.instagram.com/web/search/topsearch/?context=blended&count=50&query="
-                    + UrlEncoder.encodeUrl(params[0])).openConnection();
-            conn.setUseCaches(false);
-            conn.connect();
-
-            if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) {
-                final JSONObject jsonObject = new JSONObject(NetworkUtils.readFromConnection(conn));
-                conn.disconnect();
-
-                final JSONArray usersArray = jsonObject.getJSONArray("users");
-                final JSONArray hashtagsArray = jsonObject.getJSONArray("hashtags");
-                final JSONArray placesArray = jsonObject.getJSONArray("places");
-
-                final int usersLen = usersArray.length();
-                final int hashtagsLen = hashtagsArray.length();
-                final int placesLen = placesArray.length();
-
-                final ArrayList<SuggestionModel> suggestionModels = new ArrayList<>(usersLen + hashtagsLen);
-                for (int i = 0; i < hashtagsLen; i++) {
-                    final JSONObject hashtagsArrayJSONObject = hashtagsArray.getJSONObject(i);
-
-                    final JSONObject hashtag = hashtagsArrayJSONObject.getJSONObject("hashtag");
-
-                    suggestionModels.add(new SuggestionModel(false,
-                            hashtag.getString(Constants.EXTRAS_NAME),
-                            null,
-                            hashtag.optString("profile_pic_url", Constants.DEFAULT_HASH_TAG_PIC),
-                            SuggestionType.TYPE_HASHTAG,
-                            hashtagsArrayJSONObject.optInt("position", suggestionModels.size() - 1)));
-                }
-
-                for (int i = 0; i < placesLen; i++) {
-                    final JSONObject placesArrayJSONObject = placesArray.getJSONObject(i);
-
-                    final JSONObject place = placesArrayJSONObject.getJSONObject("place");
-
-                    // name
-                    suggestionModels.add(new SuggestionModel(false,
-                            place.getJSONObject("location").getString("pk"),  // +"/"+place.getString("slug"),
-                            place.getString("title"),
-                            place.optString("profile_pic_url"),
-                            SuggestionType.TYPE_LOCATION,
-                            placesArrayJSONObject.optInt("position", suggestionModels.size() - 1)));
-                }
-
-                for (int i = 0; i < usersLen; i++) {
-                    final JSONObject usersArrayJSONObject = usersArray.getJSONObject(i);
-
-                    final JSONObject user = usersArrayJSONObject.getJSONObject(Constants.EXTRAS_USER);
-
-                    suggestionModels.add(new SuggestionModel(user.getBoolean("is_verified"),
-                            user.getString(Constants.EXTRAS_USERNAME),
-                            user.getString("full_name"),
-                            user.getString("profile_pic_url"),
-                            SuggestionType.TYPE_USER,
-                            usersArrayJSONObject.optInt("position", suggestionModels.size() - 1)));
-                }
-
-                suggestionModels.trimToSize();
-
-                Collections.sort(suggestionModels);
-
-                result = suggestionModels.toArray(new SuggestionModel[0]);
-            }
-        } catch (final Exception e) {
-            if (BuildConfig.DEBUG && !(e instanceof InterruptedIOException)) Log.e("AWAISKING_APP", "", e);
-        }
-        return result;
-    }
-
-    @Override
-    protected void onPostExecute(final SuggestionModel[] result) {
-        if (fetchListener != null) fetchListener.onResult(result);
-    }
-}

app/src/main/java/awais/instagrabber/repositories/SearchRepository.java

@@ -0,0 +1,15 @@
+package awais.instagrabber.repositories;
+
+import java.util.Map;
+
+import awais.instagrabber.repositories.responses.search.SearchResponse;
+
+import retrofit2.Call;
+import retrofit2.http.GET;
+import retrofit2.http.QueryMap;
+import retrofit2.http.Url;
+
+public interface SearchRepository {
+    @GET
+    Call<SearchResponse> search(@Url String url, @QueryMap(encoded = true) Map<String, String> queryParams);
+}

app/src/main/java/awais/instagrabber/repositories/responses/Hashtag.java

@@ -5,19 +5,22 @@ import java.io.Serializable;
 import awais.instagrabber.models.enums.FollowingType;
 
 public final class Hashtag implements Serializable {
-    private final FollowingType following; // 0 false 1 true
+    private final FollowingType following; // 0 false 1 true; not on search results
     private final long mediaCount;
     private final String id;
     private final String name;
+    private final String searchResultSubtitle; // shows how many posts there are on search results
 
     public Hashtag(final String id,
                    final String name,
                    final long mediaCount,
-                   final FollowingType following) {
+                   final FollowingType following,
+                   final String searchResultSubtitle) {
         this.id = id;
         this.name = name;
         this.mediaCount = mediaCount;
         this.following = following;
+        this.searchResultSubtitle = searchResultSubtitle;
     }
 
     public String getId() {
@@ -35,4 +38,8 @@ public final class Hashtag implements Serializable {
     public FollowingType getFollowing() {
         return following;
     }
+
+    public String getSubtitle() {
+        return searchResultSubtitle;
+    }
 }

app/src/main/java/awais/instagrabber/repositories/responses/search/Place.java

@@ -0,0 +1,36 @@
+package awais.instagrabber.repositories.responses.search;
+
+import awais.instagrabber.repositories.responses.Location;
+
+public class Place {
+    private final Location location;
+    private final String title; // those are repeated within location
+    private final String subtitle; // address
+    private final String slug; // browser only; for end of address
+
+    public Place(final Location location,
+                 final String title,
+                 final String subtitle,
+                 final String slug) {
+        this.location = location;
+        this.title = title;
+        this.subtitle = subtitle;
+        this.slug = slug;
+    }
+
+    public Location getLocation() {
+        return location;
+    }
+
+    public String getTitle() {
+        return title;
+    }
+
+    public String getSubtitle() {
+        return subtitle;
+    }
+
+    public String getSlug() {
+        return slug;
+    }
+}

app/src/main/java/awais/instagrabber/repositories/responses/search/SearchItem.java

@@ -0,0 +1,39 @@
+package awais.instagrabber.repositories.responses.search;
+
+import java.util.List;
+
+import awais.instagrabber.repositories.responses.Hashtag;
+import awais.instagrabber.repositories.responses.User;
+
+public class SearchItem {
+    private final User user;
+    private final Place place;
+    private final Hashtag hashtag;
+    private final int position;
+
+    public SearchItem(final User user,
+                      final Place place,
+                      final Hashtag hashtag,
+                      final int position) {
+        this.user = user;
+        this.place = place;
+        this.hashtag = hashtag;
+        this.position = position;
+    }
+
+    public User getUser() {
+        return user;
+    }
+
+    public Place getPlace() {
+        return place;
+    }
+
+    public Hashtag getHashtag() {
+        return hashtag;
+    }
+
+    public int getPosition() {
+        return position;
+    }
+}

app/src/main/java/awais/instagrabber/repositories/responses/search/SearchResponse.java

@@ -0,0 +1,48 @@
+package awais.instagrabber.repositories.responses.search;
+
+import java.util.List;
+
+import awais.instagrabber.repositories.responses.User;
+
+public class SearchResponse {
+    // app
+    private final List<SearchItem> list;
+    // browser
+    private final List<SearchItem> users;
+    private final List<SearchItem> places;
+    private final List<SearchItem> hashtags;
+    // universal
+    private final String status;
+
+    public SearchResponse(final List<SearchItem> list,
+                          final List<SearchItem> users,
+                          final List<SearchItem> places,
+                          final List<SearchItem> hashtags,
+                          final String status) {
+        this.list = list;
+        this.users = users;
+        this.places = places;
+        this.hashtags = hashtags;
+        this.status = status;
+    }
+
+    public List<SearchItem> getList() {
+        return list;
+    }
+
+    public List<SearchItem> getUsers() {
+        return users;
+    }
+
+    public List<SearchItem> getPlaces() {
+        return places;
+    }
+
+    public List<SearchItem> getHashtags() {
+        return hashtags;
+    }
+
+    public String getStatus() {
+        return status;
+    }
+}

app/src/main/java/awais/instagrabber/webservices/GraphQLService.java

@@ -394,9 +394,9 @@ public class GraphQLService extends BaseService {
                     callback.onSuccess(new Hashtag(
                             body.getString(Constants.EXTRAS_ID),
                             body.getString("name"),
-                            body.getString("profile_pic_url"),
                             timelineMedia.getLong("count"),
-                            body.optBoolean("is_following") ? FollowingType.FOLLOWING : FollowingType.NOT_FOLLOWING));
+                            body.optBoolean("is_following") ? FollowingType.FOLLOWING : FollowingType.NOT_FOLLOWING,
+                            null));
                 } catch (JSONException e) {
                     Log.e(TAG, "onResponse", e);
                     if (callback != null) {

app/src/main/java/awais/instagrabber/webservices/SearchService.java

@@ -0,0 +1,50 @@
+package awais.instagrabber.webservices;
+
+import androidx.annotation.NonNull;
+
+import com.google.common.collect.ImmutableMap;
+
+import awais.instagrabber.repositories.SearchRepository;
+import awais.instagrabber.repositories.responses.search.SearchResponse;
+import awais.instagrabber.utils.TextUtils;
+import retrofit2.Call;
+import retrofit2.Callback;
+import retrofit2.Response;
+import retrofit2.Retrofit;
+
+public class SearchService extends BaseService {
+    private static final String TAG = "LocationService";
+
+    private final SearchRepository repository;
+
+    private static SearchService instance;
+
+    private SearchService() {
+        final Retrofit retrofit = getRetrofitBuilder()
+                .baseUrl("https://www.instagram.com")
+                .build();
+        repository = retrofit.create(SearchRepository.class);
+    }
+
+    public static SearchService getInstance() {
+        if (instance == null) {
+            instance = new SearchService();
+        }
+        return instance;
+    }
+
+    public Call<SearchResponse> search(final boolean isLoggedIn,
+                                       final String query,
+                                       final String context) {
+        final ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+        builder.put("query", query);
+        // context is one of: "blended", "user", "place", "hashtag"
+        // note that "place" and "hashtag" can contain ONE user result, who knows why
+        builder.put("context", context);
+        builder.put("count", "50");
+        return repository.search(isLoggedIn
+                ? "https://i.instagram.com/api/v1/fbsearch/topsearch_flat/"
+                : "https://www.instagram.com/web/search/topsearch/",
+                builder.build());
+    }
+}