update search

setup.py

@@ -13,7 +13,7 @@ install_requires = [
 
 setup(
     name="twitter-api-client",
-    version="0.5.0",
+    version="0.5.1",
     python_requires=">=3.9.7",
     description="Twitter API",
     long_description=dedent('''

twitter/config/settings.py

@@ -134,3 +134,40 @@ search_config = {
     "include_ext_edit_control": "true",
     "ext": "mediaStats,highlightedLabel,hasNftAvatar,voiceInfo,birdwatchPivot,enrichments,superFollowMetadata,unmentionInfo,editControl,collab_control,vibe"
 }
+
+trending_params = {
+    "include_profile_interstitial_type": "1",
+    "include_blocking": "1",
+    "include_blocked_by": "1",
+    "include_followed_by": "1",
+    "include_want_retweets": "1",
+    "include_mute_edge": "1",
+    "include_can_dm": "1",
+    "include_can_media_tag": "1",
+    "include_ext_has_nft_avatar": "1",
+    "include_ext_is_blue_verified": "1",
+    "include_ext_verified_type": "1",
+    "skip_status": "1",
+    "cards_platform": "Web-12",
+    "include_cards": "1",
+    "include_ext_alt_text": "true",
+    "include_ext_limited_action_results": "false",
+    "include_quote_count": "true",
+    "include_reply_count": "1",
+    "tweet_mode": "extended",
+    "include_ext_views": "true",
+    "include_entities": "true",
+    "include_user_entities": "true",
+    "include_ext_media_color": "true",
+    "include_ext_media_availability": "true",
+    "include_ext_sensitive_media_warning": "true",
+    "include_ext_trusted_friends_metadata": "true",
+    "send_error_codes": "true",
+    "simple_quoted_tweet": "true",
+    "count": 1000,
+    "requestContext": "launch",
+    "include_page_configuration": "true",
+    "initial_tab_id": "trending",
+    "entity_tokens": "false",
+    "ext": "mediaStats,highlightedLabel,hasNftAvatar,voiceInfo,birdwatchPivot,enrichments,superFollowMetadata,unmentionInfo,editControl,vibe"
+}

twitter/search.py

@@ -7,13 +7,13 @@ import random
 import re
 import time
 from pathlib import Path
-from urllib.parse import quote, urlencode, parse_qs, urlsplit, urlunsplit
 
 import aiohttp
 import requests
 
 from .config.log import log_config
 from .config.settings import search_config
+from .utils import set_qs
 
 IN_PATH = Path('~/data/raw').expanduser()
 OUT_PATH = Path(f'~/data/processed/search_results_{time.time_ns()}.json').expanduser()
@@ -86,7 +86,7 @@ async def backoff(fn, info, retries=12):
 
 
 async def get(session: aiohttp.ClientSession, api: str, params: dict) -> tuple[dict, str]:
-    url = set_qs(api, params, update=True)
+    url = set_qs(api, params, update=True, safe='()')
     r = await session.get(url)
     data = await r.json()
     next_cursor = get_cursor(data)
@@ -108,11 +108,6 @@ def get_cursor(res: dict):
         logger.debug(e)
 
 
-def set_qs(url: str, qs: dict, update=False) -> str:
-    *_, q, f = urlsplit(url)
-    return urlunsplit((*_, urlencode(qs | parse_qs(q) if update else qs, doseq=True, quote_via=quote, safe='()'), f))
-
-
 def __get_headers(fname: str = None) -> dict:
     if fname:
         with open(fname) as fp:

twitter/utils.py

@@ -1,6 +1,13 @@
+from urllib.parse import urlsplit, urlencode, urlunsplit, parse_qs, quote
+
 import ujson
 
 
+def set_qs(url: str, qs: dict, update=False, **kwargs) -> str:
+    *_, q, f = urlsplit(url)
+    return urlunsplit((*_, urlencode(qs | parse_qs(q) if update else qs, doseq=True, quote_via=quote, safe=kwargs.get('safe','')), f))
+
+
 def find_key(obj: any, key: str) -> list:
     """
     Find all values of a given key within a nested dict or list of dicts