From 1cee2cd4c07cd34ce460f5183f42f6c290d01b46 Mon Sep 17 00:00:00 2001 From: Austin Huang Date: Mon, 22 Mar 2021 16:48:35 -0400 Subject: [PATCH] improve search 1. separate logged-in and anonymous endpoints 2. migrate to retrofit + gson, retire SuggestionsFetcher 3. prefixing search with @ or # will only return users or hashtags, respectively 4. add subtitles for locations (address) and hashtags (rough post count) --- .../instagrabber/activities/MainActivity.java | 113 ++++++++++++------ .../adapters/SuggestionsAdapter.java | 26 ++-- .../asyncs/SuggestionsFetcher.java | 113 ------------------ .../repositories/SearchRepository.java | 15 +++ .../repositories/responses/Hashtag.java | 11 +- .../repositories/responses/search/Place.java | 36 ++++++ .../responses/search/SearchItem.java | 39 ++++++ .../responses/search/SearchResponse.java | 48 ++++++++ .../webservices/GraphQLService.java | 4 +- .../webservices/SearchService.java | 50 ++++++++ 10 files changed, 283 insertions(+), 172 deletions(-) delete mode 100755 app/src/main/java/awais/instagrabber/asyncs/SuggestionsFetcher.java create mode 100644 app/src/main/java/awais/instagrabber/repositories/SearchRepository.java create mode 100644 app/src/main/java/awais/instagrabber/repositories/responses/search/Place.java create mode 100644 app/src/main/java/awais/instagrabber/repositories/responses/search/SearchItem.java create mode 100644 app/src/main/java/awais/instagrabber/repositories/responses/search/SearchResponse.java create mode 100644 app/src/main/java/awais/instagrabber/webservices/SearchService.java diff --git a/app/src/main/java/awais/instagrabber/activities/MainActivity.java b/app/src/main/java/awais/instagrabber/activities/MainActivity.java index 8328d3c3..e279b54e 100644 --- a/app/src/main/java/awais/instagrabber/activities/MainActivity.java +++ b/app/src/main/java/awais/instagrabber/activities/MainActivity.java @@ -11,7 +11,6 @@ import android.content.ServiceConnection; import android.content.res.TypedArray; import android.database.MatrixCursor; import android.net.Uri; -import android.os.AsyncTask; import android.os.Build; import android.os.Bundle; import android.os.Handler; @@ -57,21 +56,22 @@ import java.util.Deque; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import awais.instagrabber.R; import awais.instagrabber.adapters.SuggestionsAdapter; import awais.instagrabber.asyncs.PostFetcher; -import awais.instagrabber.asyncs.SuggestionsFetcher; import awais.instagrabber.customviews.emoji.EmojiVariantManager; import awais.instagrabber.databinding.ActivityMainBinding; import awais.instagrabber.fragments.PostViewV2Fragment; import awais.instagrabber.fragments.directmessages.DirectMessageInboxFragmentDirections; import awais.instagrabber.fragments.main.FeedFragment; import awais.instagrabber.fragments.settings.PreferenceKeys; -import awais.instagrabber.interfaces.FetchListener; import awais.instagrabber.models.IntentModel; import awais.instagrabber.models.SuggestionModel; import awais.instagrabber.models.enums.SuggestionType; +import awais.instagrabber.repositories.responses.search.SearchItem; +import awais.instagrabber.repositories.responses.search.SearchResponse; import awais.instagrabber.services.ActivityCheckerService; import awais.instagrabber.services.DMSyncAlarmReceiver; import awais.instagrabber.utils.AppExecutors; @@ -83,6 +83,10 @@ import awais.instagrabber.utils.TextUtils; import awais.instagrabber.utils.Utils; import awais.instagrabber.utils.emoji.EmojiParser; import awais.instagrabber.viewmodels.AppStateViewModel; +import awais.instagrabber.webservices.SearchService; +import retrofit2.Call; +import retrofit2.Callback; +import retrofit2.Response; import static awais.instagrabber.utils.NavigationExtensions.setupWithNavController; import static awais.instagrabber.utils.Utils.settingsHelper; @@ -105,6 +109,7 @@ public class MainActivity extends BaseLanguageActivity implements FragmentManage private SuggestionsAdapter suggestionAdapter; private AutoCompleteTextView searchAutoComplete; private SearchView searchView; + private SearchService searchService; private boolean showSearch = true; private Handler suggestionsFetchHandler; private int firstFragmentGraphIndex; @@ -175,6 +180,7 @@ public class MainActivity extends BaseLanguageActivity implements FragmentManage EmojiVariantManager.getInstance(); }); initEmojiCompat(); + searchService = SearchService.getInstance(); // initDmService(); } @@ -338,51 +344,84 @@ public class MainActivity extends BaseLanguageActivity implements FragmentManage searchView.setOnQueryTextListener(new SearchView.OnQueryTextListener() { private boolean searchUser; private boolean searchHash; - private AsyncTask prevSuggestionAsync; + private Call prevSuggestionAsync; private final String[] COLUMNS = { BaseColumns._ID, Constants.EXTRAS_USERNAME, Constants.EXTRAS_NAME, Constants.EXTRAS_TYPE, + "query", "pfp", "verified" }; private String currentSearchQuery; - private final FetchListener fetchListener = new FetchListener() { + private final Callback cb = new Callback() { @Override - public void doBefore() { - suggestionAdapter.changeCursor(null); - } - - @Override - public void onResult(final SuggestionModel[] result) { + public void onResponse(@NonNull final Call call, + @NonNull final Response response) { final MatrixCursor cursor; - if (result == null) cursor = null; + final SearchResponse body = response.body(); + if (body == null) { + cursor = null; + return; + } + final List result = new ArrayList(); + if (isLoggedIn) { + if (body.getList() != null) result.addAll(searchHash ? body.getList() + .stream() + .filter(i -> i.getUser() == null) + .collect(Collectors.toList()) : body.getList()); + } else { - cursor = new MatrixCursor(COLUMNS, 0); - for (int i = 0; i < result.length; i++) { - final SuggestionModel suggestionModel = result[i]; - if (suggestionModel != null) { - final SuggestionType suggestionType = suggestionModel.getSuggestionType(); - final Object[] objects = { - i, - suggestionType == SuggestionType.TYPE_LOCATION ? suggestionModel.getName() : suggestionModel.getUsername(), - suggestionType == SuggestionType.TYPE_LOCATION ? suggestionModel.getUsername() : suggestionModel.getName(), - suggestionType, - suggestionModel.getProfilePic(), - suggestionModel.isVerified()}; - if (!searchHash && !searchUser) cursor.addRow(objects); - else { - final boolean isCurrHash = suggestionType == SuggestionType.TYPE_HASHTAG; - if (searchHash && isCurrHash || !searchHash && !isCurrHash) - cursor.addRow(objects); - } - } + if (body.getUsers() != null && !searchHash) result.addAll(body.getUsers()); + if (body.getHashtags() != null) result.addAll(body.getHashtags()); + if (body.getPlaces() != null) result.addAll(body.getPlaces()); + } + cursor = new MatrixCursor(COLUMNS, 0); + for (int i = 0; i < result.size(); i++) { + final SearchItem suggestionModel = result.get(i); + if (suggestionModel != null) { + Object[] objects = null; + if (suggestionModel.getUser() != null) + objects = new Object[]{ + suggestionModel.getPosition(), + suggestionModel.getUser().getUsername(), + suggestionModel.getUser().getFullName(), + SuggestionType.TYPE_USER, + suggestionModel.getUser().getUsername(), + suggestionModel.getUser().getProfilePicUrl(), + suggestionModel.getUser().isVerified()}; + else if (suggestionModel.getHashtag() != null) + objects = new Object[]{ + suggestionModel.getPosition(), + suggestionModel.getHashtag().getName(), + suggestionModel.getHashtag().getSubtitle(), + SuggestionType.TYPE_HASHTAG, + suggestionModel.getHashtag().getName(), + "res:/" + R.drawable.ic_hashtag, + false}; + else if (suggestionModel.getPlace() != null) + objects = new Object[]{ + suggestionModel.getPosition(), + suggestionModel.getPlace().getTitle(), + suggestionModel.getPlace().getSubtitle(), + SuggestionType.TYPE_LOCATION, + suggestionModel.getPlace().getLocation().getPk(), + "res:/" + R.drawable.ic_location, + false}; + cursor.addRow(objects); } } suggestionAdapter.changeCursor(cursor); } + + @Override + public void onFailure(@NonNull final Call call, + Throwable t) { + if (!call.isCanceled() && t != null) + Log.e(TAG, "Exception on search:", t); + } }; private final Runnable runnable = () -> { @@ -401,17 +440,19 @@ public class MainActivity extends BaseLanguageActivity implements FragmentManage if (searchAutoComplete != null) { searchAutoComplete.setThreshold(1); } - prevSuggestionAsync = new SuggestionsFetcher(fetchListener).executeOnExecutor( - AsyncTask.THREAD_POOL_EXECUTOR, - searchUser || searchHash ? currentSearchQuery.substring(1) - : currentSearchQuery); + prevSuggestionAsync = searchService.search(isLoggedIn, + searchUser || searchHash ? currentSearchQuery.substring(1) + : currentSearchQuery, + searchUser ? "user" : (searchHash ? "hashtag" : "blended")); + suggestionAdapter.changeCursor(null); + prevSuggestionAsync.enqueue(cb); } }; private void cancelSuggestionsAsync() { if (prevSuggestionAsync != null) try { - prevSuggestionAsync.cancel(true); + prevSuggestionAsync.cancel(); } catch (final Exception ignored) {} } diff --git a/app/src/main/java/awais/instagrabber/adapters/SuggestionsAdapter.java b/app/src/main/java/awais/instagrabber/adapters/SuggestionsAdapter.java index 154e83f2..6c4fa4c7 100755 --- a/app/src/main/java/awais/instagrabber/adapters/SuggestionsAdapter.java +++ b/app/src/main/java/awais/instagrabber/adapters/SuggestionsAdapter.java @@ -35,12 +35,12 @@ public final class SuggestionsAdapter extends CursorAdapter { @Override public void bindView(@NonNull final View view, final Context context, @NonNull final Cursor cursor) { - // i, username, fullname, type, picUrl, verified - // 0, 1 , 2 , 3 , 4 , 5 + // i, username, fullname, type, query, picUrl, verified + // 0, 1 , 2 , 3 , 4 , 5 , 6 final String fullName = cursor.getString(2); String username = cursor.getString(1); - String picUrl = cursor.getString(4); - final boolean verified = cursor.getString(5).charAt(0) == 't'; + String picUrl = cursor.getString(5); + final boolean verified = cursor.getString(6).charAt(0) == 't'; final String type = cursor.getString(3); SuggestionType suggestionType = null; @@ -50,22 +50,14 @@ public final class SuggestionsAdapter extends CursorAdapter { Log.e(TAG, "Unknown suggestion type: " + type, e); } if (suggestionType == null) return; - final String query; + String query = cursor.getString(4); switch (suggestionType) { case TYPE_USER: username = '@' + username; - query = username; break; case TYPE_HASHTAG: username = '#' + username; - query = username; break; - case TYPE_LOCATION: - query = fullName; - picUrl = "res:/" + R.drawable.ic_location; - break; - default: - return; // will never come here } if (onSuggestionClickListener != null) { @@ -75,12 +67,8 @@ public final class SuggestionsAdapter extends CursorAdapter { final ItemSuggestionBinding binding = ItemSuggestionBinding.bind(view); binding.isVerified.setVisibility(verified ? View.VISIBLE : View.GONE); binding.tvUsername.setText(username); - if (suggestionType.equals(SuggestionType.TYPE_LOCATION)) { - binding.tvFullName.setVisibility(View.GONE); - } else { - binding.tvFullName.setVisibility(View.VISIBLE); - binding.tvFullName.setText(fullName); - } + binding.tvFullName.setVisibility(View.VISIBLE); + binding.tvFullName.setText(fullName); binding.ivProfilePic.setImageURI(picUrl); } diff --git a/app/src/main/java/awais/instagrabber/asyncs/SuggestionsFetcher.java b/app/src/main/java/awais/instagrabber/asyncs/SuggestionsFetcher.java deleted file mode 100755 index c40d0406..00000000 --- a/app/src/main/java/awais/instagrabber/asyncs/SuggestionsFetcher.java +++ /dev/null @@ -1,113 +0,0 @@ -package awais.instagrabber.asyncs; - -import android.os.AsyncTask; -import android.util.Log; - -import org.json.JSONArray; -import org.json.JSONObject; - -import java.io.InterruptedIOException; -import java.net.HttpURLConnection; -import java.net.URL; -import java.util.ArrayList; -import java.util.Collections; - -import awais.instagrabber.BuildConfig; -import awais.instagrabber.interfaces.FetchListener; -import awais.instagrabber.models.SuggestionModel; -import awais.instagrabber.models.enums.SuggestionType; -import awais.instagrabber.utils.Constants; -import awais.instagrabber.utils.NetworkUtils; -import awais.instagrabber.utils.UrlEncoder; - -public final class SuggestionsFetcher extends AsyncTask { - private final FetchListener fetchListener; - - public SuggestionsFetcher(final FetchListener fetchListener) { - this.fetchListener = fetchListener; - } - - @Override - protected void onPreExecute() { - if (fetchListener != null) fetchListener.doBefore(); - } - - @Override - protected SuggestionModel[] doInBackground(final String... params) { - SuggestionModel[] result = null; - try { - final HttpURLConnection conn = (HttpURLConnection) new URL("https://www.instagram.com/web/search/topsearch/?context=blended&count=50&query=" - + UrlEncoder.encodeUrl(params[0])).openConnection(); - conn.setUseCaches(false); - conn.connect(); - - if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) { - final JSONObject jsonObject = new JSONObject(NetworkUtils.readFromConnection(conn)); - conn.disconnect(); - - final JSONArray usersArray = jsonObject.getJSONArray("users"); - final JSONArray hashtagsArray = jsonObject.getJSONArray("hashtags"); - final JSONArray placesArray = jsonObject.getJSONArray("places"); - - final int usersLen = usersArray.length(); - final int hashtagsLen = hashtagsArray.length(); - final int placesLen = placesArray.length(); - - final ArrayList suggestionModels = new ArrayList<>(usersLen + hashtagsLen); - for (int i = 0; i < hashtagsLen; i++) { - final JSONObject hashtagsArrayJSONObject = hashtagsArray.getJSONObject(i); - - final JSONObject hashtag = hashtagsArrayJSONObject.getJSONObject("hashtag"); - - suggestionModels.add(new SuggestionModel(false, - hashtag.getString(Constants.EXTRAS_NAME), - null, - hashtag.optString("profile_pic_url", Constants.DEFAULT_HASH_TAG_PIC), - SuggestionType.TYPE_HASHTAG, - hashtagsArrayJSONObject.optInt("position", suggestionModels.size() - 1))); - } - - for (int i = 0; i < placesLen; i++) { - final JSONObject placesArrayJSONObject = placesArray.getJSONObject(i); - - final JSONObject place = placesArrayJSONObject.getJSONObject("place"); - - // name - suggestionModels.add(new SuggestionModel(false, - place.getJSONObject("location").getString("pk"), // +"/"+place.getString("slug"), - place.getString("title"), - place.optString("profile_pic_url"), - SuggestionType.TYPE_LOCATION, - placesArrayJSONObject.optInt("position", suggestionModels.size() - 1))); - } - - for (int i = 0; i < usersLen; i++) { - final JSONObject usersArrayJSONObject = usersArray.getJSONObject(i); - - final JSONObject user = usersArrayJSONObject.getJSONObject(Constants.EXTRAS_USER); - - suggestionModels.add(new SuggestionModel(user.getBoolean("is_verified"), - user.getString(Constants.EXTRAS_USERNAME), - user.getString("full_name"), - user.getString("profile_pic_url"), - SuggestionType.TYPE_USER, - usersArrayJSONObject.optInt("position", suggestionModels.size() - 1))); - } - - suggestionModels.trimToSize(); - - Collections.sort(suggestionModels); - - result = suggestionModels.toArray(new SuggestionModel[0]); - } - } catch (final Exception e) { - if (BuildConfig.DEBUG && !(e instanceof InterruptedIOException)) Log.e("AWAISKING_APP", "", e); - } - return result; - } - - @Override - protected void onPostExecute(final SuggestionModel[] result) { - if (fetchListener != null) fetchListener.onResult(result); - } -} \ No newline at end of file diff --git a/app/src/main/java/awais/instagrabber/repositories/SearchRepository.java b/app/src/main/java/awais/instagrabber/repositories/SearchRepository.java new file mode 100644 index 00000000..7dda390d --- /dev/null +++ b/app/src/main/java/awais/instagrabber/repositories/SearchRepository.java @@ -0,0 +1,15 @@ +package awais.instagrabber.repositories; + +import java.util.Map; + +import awais.instagrabber.repositories.responses.search.SearchResponse; + +import retrofit2.Call; +import retrofit2.http.GET; +import retrofit2.http.QueryMap; +import retrofit2.http.Url; + +public interface SearchRepository { + @GET + Call search(@Url String url, @QueryMap(encoded = true) Map queryParams); +} diff --git a/app/src/main/java/awais/instagrabber/repositories/responses/Hashtag.java b/app/src/main/java/awais/instagrabber/repositories/responses/Hashtag.java index 17bc3c63..2ce08eb5 100755 --- a/app/src/main/java/awais/instagrabber/repositories/responses/Hashtag.java +++ b/app/src/main/java/awais/instagrabber/repositories/responses/Hashtag.java @@ -5,19 +5,22 @@ import java.io.Serializable; import awais.instagrabber.models.enums.FollowingType; public final class Hashtag implements Serializable { - private final FollowingType following; // 0 false 1 true + private final FollowingType following; // 0 false 1 true; not on search results private final long mediaCount; private final String id; private final String name; + private final String searchResultSubtitle; // shows how many posts there are on search results public Hashtag(final String id, final String name, final long mediaCount, - final FollowingType following) { + final FollowingType following, + final String searchResultSubtitle) { this.id = id; this.name = name; this.mediaCount = mediaCount; this.following = following; + this.searchResultSubtitle = searchResultSubtitle; } public String getId() { @@ -35,4 +38,8 @@ public final class Hashtag implements Serializable { public FollowingType getFollowing() { return following; } + + public String getSubtitle() { + return searchResultSubtitle; + } } \ No newline at end of file diff --git a/app/src/main/java/awais/instagrabber/repositories/responses/search/Place.java b/app/src/main/java/awais/instagrabber/repositories/responses/search/Place.java new file mode 100644 index 00000000..e4a44646 --- /dev/null +++ b/app/src/main/java/awais/instagrabber/repositories/responses/search/Place.java @@ -0,0 +1,36 @@ +package awais.instagrabber.repositories.responses.search; + +import awais.instagrabber.repositories.responses.Location; + +public class Place { + private final Location location; + private final String title; // those are repeated within location + private final String subtitle; // address + private final String slug; // browser only; for end of address + + public Place(final Location location, + final String title, + final String subtitle, + final String slug) { + this.location = location; + this.title = title; + this.subtitle = subtitle; + this.slug = slug; + } + + public Location getLocation() { + return location; + } + + public String getTitle() { + return title; + } + + public String getSubtitle() { + return subtitle; + } + + public String getSlug() { + return slug; + } +} diff --git a/app/src/main/java/awais/instagrabber/repositories/responses/search/SearchItem.java b/app/src/main/java/awais/instagrabber/repositories/responses/search/SearchItem.java new file mode 100644 index 00000000..52749321 --- /dev/null +++ b/app/src/main/java/awais/instagrabber/repositories/responses/search/SearchItem.java @@ -0,0 +1,39 @@ +package awais.instagrabber.repositories.responses.search; + +import java.util.List; + +import awais.instagrabber.repositories.responses.Hashtag; +import awais.instagrabber.repositories.responses.User; + +public class SearchItem { + private final User user; + private final Place place; + private final Hashtag hashtag; + private final int position; + + public SearchItem(final User user, + final Place place, + final Hashtag hashtag, + final int position) { + this.user = user; + this.place = place; + this.hashtag = hashtag; + this.position = position; + } + + public User getUser() { + return user; + } + + public Place getPlace() { + return place; + } + + public Hashtag getHashtag() { + return hashtag; + } + + public int getPosition() { + return position; + } +} diff --git a/app/src/main/java/awais/instagrabber/repositories/responses/search/SearchResponse.java b/app/src/main/java/awais/instagrabber/repositories/responses/search/SearchResponse.java new file mode 100644 index 00000000..04db0286 --- /dev/null +++ b/app/src/main/java/awais/instagrabber/repositories/responses/search/SearchResponse.java @@ -0,0 +1,48 @@ +package awais.instagrabber.repositories.responses.search; + +import java.util.List; + +import awais.instagrabber.repositories.responses.User; + +public class SearchResponse { + // app + private final List list; + // browser + private final List users; + private final List places; + private final List hashtags; + // universal + private final String status; + + public SearchResponse(final List list, + final List users, + final List places, + final List hashtags, + final String status) { + this.list = list; + this.users = users; + this.places = places; + this.hashtags = hashtags; + this.status = status; + } + + public List getList() { + return list; + } + + public List getUsers() { + return users; + } + + public List getPlaces() { + return places; + } + + public List getHashtags() { + return hashtags; + } + + public String getStatus() { + return status; + } +} diff --git a/app/src/main/java/awais/instagrabber/webservices/GraphQLService.java b/app/src/main/java/awais/instagrabber/webservices/GraphQLService.java index 4cd1c1b6..b373b3c8 100644 --- a/app/src/main/java/awais/instagrabber/webservices/GraphQLService.java +++ b/app/src/main/java/awais/instagrabber/webservices/GraphQLService.java @@ -394,9 +394,9 @@ public class GraphQLService extends BaseService { callback.onSuccess(new Hashtag( body.getString(Constants.EXTRAS_ID), body.getString("name"), - body.getString("profile_pic_url"), timelineMedia.getLong("count"), - body.optBoolean("is_following") ? FollowingType.FOLLOWING : FollowingType.NOT_FOLLOWING)); + body.optBoolean("is_following") ? FollowingType.FOLLOWING : FollowingType.NOT_FOLLOWING, + null)); } catch (JSONException e) { Log.e(TAG, "onResponse", e); if (callback != null) { diff --git a/app/src/main/java/awais/instagrabber/webservices/SearchService.java b/app/src/main/java/awais/instagrabber/webservices/SearchService.java new file mode 100644 index 00000000..39b5bd65 --- /dev/null +++ b/app/src/main/java/awais/instagrabber/webservices/SearchService.java @@ -0,0 +1,50 @@ +package awais.instagrabber.webservices; + +import androidx.annotation.NonNull; + +import com.google.common.collect.ImmutableMap; + +import awais.instagrabber.repositories.SearchRepository; +import awais.instagrabber.repositories.responses.search.SearchResponse; +import awais.instagrabber.utils.TextUtils; +import retrofit2.Call; +import retrofit2.Callback; +import retrofit2.Response; +import retrofit2.Retrofit; + +public class SearchService extends BaseService { + private static final String TAG = "LocationService"; + + private final SearchRepository repository; + + private static SearchService instance; + + private SearchService() { + final Retrofit retrofit = getRetrofitBuilder() + .baseUrl("https://www.instagram.com") + .build(); + repository = retrofit.create(SearchRepository.class); + } + + public static SearchService getInstance() { + if (instance == null) { + instance = new SearchService(); + } + return instance; + } + + public Call search(final boolean isLoggedIn, + final String query, + final String context) { + final ImmutableMap.Builder builder = ImmutableMap.builder(); + builder.put("query", query); + // context is one of: "blended", "user", "place", "hashtag" + // note that "place" and "hashtag" can contain ONE user result, who knows why + builder.put("context", context); + builder.put("count", "50"); + return repository.search(isLoggedIn + ? "https://i.instagram.com/api/v1/fbsearch/topsearch_flat/" + : "https://www.instagram.com/web/search/topsearch/", + builder.build()); + } +}