Currently, the way to paginate searches is to get the serpapi_pagination.current and increase the offset or start parameters in the loop. Like with regular HTTP requests to serpapi.com/search without an API wrapper.

import os
from serpapi import GoogleSearch

params = {
    "engine": "google",
    "q": "coffee",
    "tbm": "nws",
    "api_key": os.getenv("API_KEY"),
}

search = GoogleSearch(params)
results = search.get_dict()

print(f"Current page: {results['serpapi_pagination']['current']}")

for news_result in results["news_results"]:
    print(f"Title: {news_result['title']}\nLink: {news_result['link']}\n")

while 'next' in results['serpapi_pagination']:
    search.params_dict[
        "start"] = results['serpapi_pagination']['current'] * 10
    results = search.get_dict()

    print(f"Current page: {results['serpapi_pagination']['current']}")

    for news_result in results["news_results"]:
        print(
            f"Title: {news_result['title']}\nLink: {news_result['link']}\n"
        )

A more convenient way for an official API wrapper would be to provide some function like search.paginate(callback: Callable) which will properly calculate offset for the specific search engine and loop through pages until the end.

import os
from serpapi import GoogleSearch

def print_results(results):
  print(f"Current page: {results['serpapi_pagination']['current']}")

  for news_result in results["news_results"]:
    print(f"Title: {news_result['title']}\nLink: {news_result['link']}\n")

params = {
    "engine": "google",
    "q": "coffee",
    "tbm": "nws",
    "api_key": os.getenv("API_KEY"),
}

search = GoogleSearch(params)
search.paginate(print_results)

@jvmvik @hartator What do you think?

I am using SERP API to fetch google scholar papers, although there is always a link called "related articles' under each article but SERP API doesn't have any SERP URL to fetch data of those links?

Serp API result :

Can I directly call this URL https://scholar.google.com/scholar?q=related:gemrYG-1WnEJ:scholar.google.com/&scioq=Multi-label+text+classification+with+latent+word-wise+label+information&hl=en&as_sdt=0,21 using serp API?

I am trying to use the pagination feature based on the code at (https://github.com/serpapi/google-search-results-python#pagination-using-iterator). I want to request 20 results per API call but pagination by default iterates by 10 results only instead of 20, meaning I my requests end up overlapping.

I think I have found a solution to this. Looking in the package, the pagination.py file has a Pagination class which takes a page_size variable that changes the size of the offset between returned results. The Pagination class is imported in the serp_api_client.py file within the pagination method starting on line 170 but here the page_size variable wasn't included. I just added page_size = 10 on lines 170 and 174 and now I can use the page_size variable if I call search.pagination(page_size = 20). Can this change be made in the code?

`from serpapi.google_search_results import GoogleSearchResults

client = GoogleSearchResults({"q": "coffee", "serp_api_key": "************************"})

result = client.get_dict()`

I tried giving my API key from serpstack. Yet I am left with this error. Any help could be much useful.

When installing the package via pip it fails.

Collecting google-search-results
  Using cached https://files.pythonhosted.org/packages/08/eb/38646304d98db83d85f57599d2ccc8caf325961e8792100a1014950197a6/google_search_results-1.5.2.tar.gz
    Complete output from command python setup.py egg_info:
    Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "/private/var/folders/3m/91gj9l890y71886_7sfndl3r0000gn/T/pip-install-YVqFKL/google-search-results/setup.py", line 7, in <module>
        with open(path.join(here, 'SHORT_README.rst'), encoding='utf-8') as f:
      File "/usr/local/Cellar/python@2/2.7.15_3/Frameworks/Python.framework/Versions/2.7/lib/python2.7/codecs.py", line 898, in open
        file = __builtin__.open(filename, mode, buffering)
    IOError: [Errno 2] No such file or directory: '/private/var/folders/3m/91gj9l890y71886_7sfndl3r0000gn/T/pip-install-YVqFKL/google-search-results/SHORT_README.rst'

    ----------------------------------------
Command "python setup.py egg_info" failed with error code 1 in /private/var/folders/3m/91gj9l890y71886_7sfndl3r0000gn/T/pip-install-YVqFKL/google-search-results/

Running macOS catalina and python 2.7

~ ❯❯❯ pip --version
pip 19.0.2 from /usr/local/lib/python2.7/site-packages/pip (python 2.7)
~ ❯❯❯ python --version
Python 2.7.15

SerpApiClient.get_search_archive assumes all results must be loaded as a JSON, so it fails when using format='html'

GoogleSearchResults({}).get_search_archive(search_id='5df0db57ab3f5837994cd5a1', format='html')

---------------------------------------------------------------------------                                                                                                                                   JSONDecodeError                           Traceback (most recent call last)
<ipython-input-8-b6d24cb47bf7> in <module>
----> 1 GoogleSearchResults({}).get_search_archive(search_id='5df0db57ab3f5837994cd5a1', format='html')

C:\ProgramData\Anaconda3\lib\site-packages\serpapi\serp_api_client.py in get_search_archive(self, search_id, format)
78             dict|string: search result from the archive
79         """
---> 80         return json.loads(self.get_results("/searches/{0}.{1}".format(search_id, format)))
81
82     def get_account(self):

C:\ProgramData\Anaconda3\lib\json\__init__.py in loads(s, encoding, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)
352             parse_int is None and parse_float is None and
353             parse_constant is None and object_pairs_hook is None and not kw):
--> 354         return _default_decoder.decode(s)
355     if cls is None:
356         cls = JSONDecoder

C:\ProgramData\Anaconda3\lib\json\decoder.py in decode(self, s, _w)
337
338         """
--> 339         obj, end = self.raw_decode(s, idx=_w(s, 0).end())
340         end = _w(s, end).end()
341         if end != len(s):

C:\ProgramData\Anaconda3\lib\json\decoder.py in raw_decode(self, s, idx)
355             obj, end = self.scan_once(s, idx)
356         except StopIteration as err:
--> 357             raise JSONDecodeError("Expecting value", s, err.value) from None
358         return obj, end

JSONDecodeError: Expecting value: line 1 column 1 (char 0)

I've attempted to get the results from the answer box using the documentation here.

I noticed the Playground does not return these results either.

Is there any way to get this URL also?

Output Returned when Attempting to Run the Sample Provided:

from serpapi import GoogleSearch

params = {
  "q": "What's the definition of transparent?",
  "hl": "en",
  "gl": "us",
  "api_key": ""
}

search = GoogleSearch(params)
results = search.get_dict()
answer_box = results['answer_box']

Hello,

I am trying to run the example on your git page https://github.com/serpapi/google-search-results-python#batch-asynchronous-searches But constantly getting this error.

Thanks

This is the error message you get when you don't supply a private key. I think information on this site should be provided regarding:

1. How to get an API key
2. Is a key free or how much does it cost
3. Are there limits to using the key (hits/hour or whatever)

The service provided by the repo is very valuable, but can I use it or not depends on the answers to these questions.

Some queries which return a knowledge graph in both my own google search and when tested in the SerpApi Playground are not returning the 'knowledge_graph' key in my own application.

Code:

params = {
    'q': 'Aspen Pumps Ltd',
    'engine': 'google',
    'api_key': <api_key>,
    'num': 100
  }

result_set = GoogleSearchResults(params).get_dict()

print(result_set.keys())

Evaluation:

dict_keys(['search_metadata', 'search_parameters', 'search_information', 'ads', 'shopping_results', 'organic_results', 'related_searches', 'pagination', 'serpapi_pagination'])

Manual Results:

https://www.google.com

https://serpapi.com/playground

Hi, One of the user using my code get the following error when creating a client. I suppose it is machine settings related as it doesn't happen to other users. Thanks for helping P.S. I am fairly new to coding.

Since the start value starts from 0, the correct second page should be 10 and not 11.

This behaviour is causing a skip in pages also. The customers are getting confusing results:

Intercom Link First recognized by @marm123.

I think this part needs to be replaced by:

self.client.params_dict['start'] += 0

Whether it would cause any error on other engines is something I don't know. But it may also fix it for every other engine.

This PR focuses on:

• fix typos.

• remove duplicates, for example:

• text formatting, clarification.

• remove unnecessary examples, or possibly leave them but with details disclosure element.

• add backlinks to some terms.

• add pagination examples for all available APIs.

Pagination examples could be placed to oobt/ folder instead to test "out of the box tests" for all APIs. However, I'm not familiar with it and how things currently work so I need a little guidance before working on it 🙂

As Google Jobs does not return serpapi_pagination key but expects start param to paginate, this iteration of the library does not support pagination in Google Jobs. Pagination Support to be added for Google Jobs.

# stop if backend miss to return serpapi_pagination
if not 'serpapi_pagination' in result:
  raise StopIteration

# stop if no next page
if not 'next' in result['serpapi_pagination']:
    raise StopIteration

start and num parameters are not suitable for token-based pagination. Such pagination is used on Google Maps, YouTube, Google Scholar Authors, and other search engines.

This PR consumes URL query parameters for the next page. It stops paginating when parameters do not change.

Details: https://github.com/serpapi/google-search-results-python/issues/22

Some tests are failing because start and num parameters are not supported anymore. These tests will be fixed in the following commits.

Related: #25, #26

Hi, A new scrapper here. in my api call, i have the following error. Would you please let me know if i am doing anything wrong here? Thanks a lot

https://serpapi.com/search
---------------------------------------------------------------------------
ConnectionResetError                      Traceback (most recent call last)
/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    676                 headers=headers,
--> 677                 chunked=chunked,
    678             )

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    380         try:
--> 381             self._validate_conn(conn)
    382         except (SocketTimeout, BaseSSLError) as e:

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connectionpool.py in _validate_conn(self, conn)
    977         if not getattr(conn, "sock", None):  # AppEngine might not have  `.sock`
--> 978             conn.connect()
    979 

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connection.py in connect(self)
    370             server_hostname=server_hostname,
--> 371             ssl_context=context,
    372         )

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/util/ssl_.py in ssl_wrap_socket(sock, keyfile, certfile, cert_reqs, ca_certs, server_hostname, ssl_version, ciphers, ssl_context, ca_cert_dir, key_password, ca_cert_data)
    385         if HAS_SNI and server_hostname is not None:
--> 386             return context.wrap_socket(sock, server_hostname=server_hostname)
    387 

/anaconda/envs/azureml_py36/lib/python3.6/ssl.py in wrap_socket(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, session)
    406                          server_hostname=server_hostname,
--> 407                          _context=self, _session=session)
    408 

/anaconda/envs/azureml_py36/lib/python3.6/ssl.py in __init__(self, sock, keyfile, certfile, server_side, cert_reqs, ssl_version, ca_certs, do_handshake_on_connect, family, type, proto, fileno, suppress_ragged_eofs, npn_protocols, ciphers, server_hostname, _context, _session)
    816                         raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
--> 817                     self.do_handshake()
    818 

/anaconda/envs/azureml_py36/lib/python3.6/ssl.py in do_handshake(self, block)
   1076                 self.settimeout(None)
-> 1077             self._sslobj.do_handshake()
   1078         finally:

/anaconda/envs/azureml_py36/lib/python3.6/ssl.py in do_handshake(self)
    688         """Start the SSL/TLS handshake."""
--> 689         self._sslobj.do_handshake()
    690         if self.context.check_hostname:

ConnectionResetError: [Errno 104] Connection reset by peer

During handling of the above exception, another exception occurred:

ProtocolError                             Traceback (most recent call last)
/anaconda/envs/azureml_py36/lib/python3.6/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    448                     retries=self.max_retries,
--> 449                     timeout=timeout
    450                 )

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    726             retries = retries.increment(
--> 727                 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
    728             )

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/util/retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
    409             if read is False or not self._is_method_retryable(method):
--> 410                 raise six.reraise(type(error), error, _stacktrace)
    411             elif read is not None:

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/packages/six.py in reraise(tp, value, tb)
    733             if value.__traceback__ is not tb:
--> 734                 raise value.with_traceback(tb)
    735             raise value

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    676                 headers=headers,
--> 677                 chunked=chunked,
    678             )

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    380         try:
--> 381             self._validate_conn(conn)
    382         except (SocketTimeout, BaseSSLError) as e:

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connectionpool.py in _validate_conn(self, conn)
    977         if not getattr(conn, "sock", None):  # AppEngine might not have  `.sock`
--> 978             conn.connect()
    979 

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connection.py in connect(self)
    370             server_hostname=server_hostname,
--> 371             ssl_context=context,
    372         )

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/util/ssl_.py in ssl_wrap_socket(sock, keyfile, certfile, cert_reqs, ca_certs, server_hostname, ssl_version, ciphers, ssl_context, ca_cert_dir, key_password, ca_cert_data)
    385         if HAS_SNI and server_hostname is not None:
--> 386             return context.wrap_socket(sock, server_hostname=server_hostname)
    387 

/anaconda/envs/azureml_py36/lib/python3.6/ssl.py in wrap_socket(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, session)
    406                          server_hostname=server_hostname,
--> 407                          _context=self, _session=session)
    408 

/anaconda/envs/azureml_py36/lib/python3.6/ssl.py in __init__(self, sock, keyfile, certfile, server_side, cert_reqs, ssl_version, ca_certs, do_handshake_on_connect, family, type, proto, fileno, suppress_ragged_eofs, npn_protocols, ciphers, server_hostname, _context, _session)
    816                         raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
--> 817                     self.do_handshake()
    818 

/anaconda/envs/azureml_py36/lib/python3.6/ssl.py in do_handshake(self, block)
   1076                 self.settimeout(None)
-> 1077             self._sslobj.do_handshake()
   1078         finally:

/anaconda/envs/azureml_py36/lib/python3.6/ssl.py in do_handshake(self)
    688         """Start the SSL/TLS handshake."""
--> 689         self._sslobj.do_handshake()
    690         if self.context.check_hostname:

ProtocolError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))

During handling of the above exception, another exception occurred:

ConnectionError                           Traceback (most recent call last)
<ipython-input-26-45ac328ca8f8> in <module>
      1 question = 'where to get best coffee'
----> 2 results = performSearch(question)

<ipython-input-25-5bc778bad4e2> in performSearch(question)
     12 
     13     search = GoogleSearch(params)
---> 14     results = search.get_dict()
     15     return results

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/serpapi/serp_api_client.py in get_dict(self)
    101             (alias for get_dictionary)
    102         """
--> 103         return self.get_dictionary()
    104 
    105     def get_object(self):

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/serpapi/serp_api_client.py in get_dictionary(self)
     94             Dict with the formatted response content
     95         """
---> 96         return dict(self.get_json())
     97 
     98     def get_dict(self):

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/serpapi/serp_api_client.py in get_json(self)
     81         """
     82         self.params_dict["output"] = "json"
---> 83         return json.loads(self.get_results())
     84 
     85     def get_raw_json(self):

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/serpapi/serp_api_client.py in get_results(self, path)
     68             Response text field
     69         """
---> 70         return self.get_response(path).text
     71 
     72     def get_html(self):

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/serpapi/serp_api_client.py in get_response(self, path)
     57             url, parameter = self.construct_url(path)
     58             print(url)
---> 59             response = requests.get(url, parameter, timeout=self.timeout)
     60             return response
     61         except requests.HTTPError as e:

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/requests/api.py in get(url, params, **kwargs)
     73     """
     74 
---> 75     return request('get', url, params=params, **kwargs)
     76 
     77 

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/requests/api.py in request(method, url, **kwargs)
     59     # cases, and look like a memory leak in others.
     60     with sessions.Session() as session:
---> 61         return session.request(method=method, url=url, **kwargs)
     62 
     63 

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
    540         }
    541         send_kwargs.update(settings)
--> 542         resp = self.send(prep, **send_kwargs)
    543 
    544         return resp

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/requests/sessions.py in send(self, request, **kwargs)
    653 
    654         # Send the request
--> 655         r = adapter.send(request, **kwargs)
    656 
    657         # Total elapsed time of the request (approximately)

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    496 
    497         except (ProtocolError, socket.error) as err:
--> 498             raise ConnectionError(err, request=request)
    499 
    500         except MaxRetryError as e:

ConnectionError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))

When retrieving results from Google Scholar using the pagination() method, the first article on the second page of google scholar is always missing.

I think this is caused by the following snippet in the update() method of google-search-results-python/serpapi/pagination.py:

def update(self):
        self.client.params_dict["start"] = self.start
        self.client.params_dict["num"] = self.num
        if self.start > 0:
            self.client.params_dict["start"] += 1

This seems to mean that for all pages except the first, paginate increases start by 1. So while for the first page it requests results starting at 0 and ending at 19 (if page_size=20). For the second page it requests results starting at 21 and ending at 40, skipping result 20.

If I delete the if statement, the code seems to work as intended and I get result 19 back.