OfferUp product url listings

# add headers and cookies

headers = {}
cookies = {}

query = """
    query GetModularFeed($searchParams: [SearchParam], $debug: Boolean = false) {\n  modularFeed(params: $searchParams, debug: $debug) {\n    analyticsData {\n      requestId\n      searchPerformedEventUniqueId\n      searchSessionId\n      __typename\n    }\n    categoryInfo {\n      categoryId\n      isForcedCategory\n      __typename\n    }\n    feedAdditions\n    filters {\n      ...modularFilterNumericRange\n      ...modularFilterSelectionList\n      __typename\n    }\n    legacyFeedOptions {\n      ...legacyFeedOptionListSelection\n      ...legacyFeedOptionNumericRange\n      __typename\n    }\n    looseTiles {\n      ...modularTileBanner\n      ...modularTileBingAd\n      ...modularTileGoogleDisplayAd\n      ...modularTileJob\n      ...modularTileEmptyState\n      ...modularTileListing\n      ...modularTileLocalDisplayAd\n      ...modularTileSearchAlert\n      ...modularTileSellerAd\n      ...modularModuleTileAdsPostXAd\n      __typename\n    }\n    modules {\n      ...modularGridModule\n      __typename\n    }\n    pageCursor\n    query {\n      ...modularQueryInfo\n      __typename\n    }\n    requestTimeMetadata {\n      resolverComputationTimeSeconds\n      serviceRequestTimeSeconds\n      totalResolverTimeSeconds\n      __typename\n    }\n    searchAlert {\n      alertId\n      alertStatus\n      searchAlertCount\n      __typename\n    }\n    personalizationPath\n    debugInformation @include(if: $debug) {\n      rankedListings {\n        listingId\n        attributes {\n          key\n          value\n          __typename\n        }\n        __typename\n      }\n      lastViewedItems {\n        listingId\n        attributes {\n          key\n          value\n          __typename\n        }\n        __typename\n      }\n      categoryAffinities {\n        affinity\n        count\n        decay\n        affinityOwner\n        __typename\n      }\n      rankingStats {\n        key\n        value\n        __typename\n      }\n      __typename\n    }\n    __typename\n  }\n}\n\nfragment modularFilterNumericRange on ModularFeedNumericRangeFilter {\n  isExpandedHighlight\n  lowerBound {\n    ...modularFilterNumericRangeBound\n    __typename\n  }\n  shortcutLabel\n  shortcutRank\n  subTitle\n  targetName\n  title\n  type\n  upperBound {\n    ...modularFilterNumericRangeBound\n    __typename\n  }\n  __typename\n}\n\nfragment modularFilterNumericRangeBound on ModularFeedNumericRangeFilterNumericRangeBound {\n  label\n  limit\n  placeholderText\n  targetName\n  value\n  __typename\n}\n\nfragment modularFilterSelectionList on ModularFeedSelectionListFilter {\n  targetName\n  title\n  subTitle\n  shortcutLabel\n  shortcutRank\n  type\n  isExpandedHighlight\n  options {\n    ...modularFilterSelectionListOption\n    __typename\n  }\n  __typename\n}\n\nfragment modularFilterSelectionListOption on ModularFeedSelectionListFilterOption {\n  isDefault\n  isSelected\n  label\n  subLabel\n  value\n  __typename\n}\n\nfragment legacyFeedOptionListSelection on FeedOptionListSelection {\n  label\n  labelShort\n  name\n  options {\n    default\n    label\n    labelShort\n    selected\n    subLabel\n    value\n    __typename\n  }\n  position\n  queryParam\n  type\n  __typename\n}\n\nfragment legacyFeedOptionNumericRange on FeedOptionNumericRange {\n  label\n  labelShort\n  leftQueryParam\n  lowerBound\n  name\n  options {\n    currentValue\n    label\n    textHint\n    __typename\n  }\n  position\n  rightQueryParam\n  type\n  units\n  upperBound\n  __typename\n}\n\nfragment modularTileBanner on ModularFeedTileBanner {\n  tileId\n  tileType\n  title\n  __typename\n}\n\nfragment modularTileBingAd on ModularFeedTileBingAd {\n  tileId\n  bingAd {\n    ouAdId\n    adExperimentId\n    adNetwork\n    adRequestId\n    adTileType\n    adSettings {\n      repeatClickRefractoryPeriodMillis\n      collapsible\n      __typename\n    }\n    bingClientId\n    clickFeedbackUrl\n    clickReturnUrl\n    contentUrl\n    deepLinkEnabled\n    experimentDataHash\n    image {\n      height\n      url\n      width\n      __typename\n    }\n    impressionFeedbackUrl\n    impressionUrls\n    viewableImpressionUrls\n    installmentInfo {\n      amount\n      description\n      downPayment\n      __typename\n    }\n    itemName\n    lowPrice\n    price\n    searchId\n    sellerName\n    templateFields {\n      key\n      value\n      __typename\n    }\n    __typename\n  }\n  tileType\n  __typename\n}\n\nfragment modularTileGoogleDisplayAd on ModularFeedTileGoogleDisplayAd {\n  tileId\n  googleDisplayAd {\n    ouAdId\n    additionalSizes\n    adExperimentId\n    adHeight\n    adNetwork\n    adPage\n    adRequestId\n    adTileType\n    adWidth\n    adaptive\n    channel\n    clickFeedbackUrl\n    clientId\n    contentUrl\n    customTargeting {\n      key\n      values\n      __typename\n    }\n    displayAdType\n    errorDrawable {\n      actionPath\n      listImage {\n        height\n        url\n        width\n        __typename\n      }\n      __typename\n    }\n    experimentDataHash\n    formatIds\n    impressionFeedbackUrl\n    personalizationProperties {\n      key\n      values\n      __typename\n    }\n    prebidConfigs {\n      key\n      values {\n        timeout\n        tamSlotUUID\n        liftoffPlacementIDs\n        nimbusPriceMapping\n        adPosition\n        __typename\n      }\n      __typename\n    }\n    renderLocation\n    searchId\n    searchQuery\n    templateId\n    __typename\n  }\n  tileType\n  __typename\n}\n\nfragment modularTileJob on ModularFeedTileJob {\n  tileId\n  tileType\n  job {\n    address {\n      city\n      state\n      zipcode\n      __typename\n    }\n    companyName\n    datePosted\n    image {\n      height\n      url\n      width\n      __typename\n    }\n    industry\n    jobId\n    jobListingUrl\n    jobOwnerId\n    pills {\n      text\n      type\n      __typename\n    }\n    title\n    apply {\n      method\n      value\n      __typename\n    }\n    wageDisplayValue\n    provider\n    __typename\n  }\n  __typename\n}\n\nfragment modularTileEmptyState on ModularFeedTileEmptyState {\n  tileId\n  tileType\n  title\n  description\n  iconType\n  __typename\n}\n\nfragment modularTileListing on ModularFeedTileListing {\n  tileId\n  listing {\n    ...modularListing\n    __typename\n  }\n  tileType\n  __typename\n}\n\nfragment modularListing on ModularFeedListing {\n  listingId\n  conditionText\n  flags\n  image {\n    height\n    url\n    width\n    __typename\n  }\n  isFirmPrice\n  locationName\n  price\n  title\n  vehicleMiles\n  __typename\n}\n\nfragment modularTileLocalDisplayAd on ModularFeedTileLocalDisplayAd {\n  tileId\n  localDisplayAd {\n    ouAdId\n    adExperimentId\n    adNetwork\n    adRequestId\n    adTileType\n    advertiserId\n    businessName\n    callToAction\n    callToActionType\n    clickFeedbackUrl\n    contentUrl\n    experimentDataHash\n    headline\n    image {\n      height\n      url\n      width\n      __typename\n    }\n    impressionFeedbackUrl\n    searchId\n    __typename\n  }\n  tileType\n  __typename\n}\n\nfragment modularTileSearchAlert on ModularFeedTileSearchAlert {\n  tileId\n  tileType\n  title\n  __typename\n}\n\nfragment modularTileSellerAd on ModularFeedTileSellerAd {\n  tileId\n  listing {\n    ...modularListing\n    __typename\n  }\n  sellerAd {\n    ouAdId\n    adId\n    adExperimentId\n    adNetwork\n    adRequestId\n    adTileType\n    clickFeedbackUrl\n    experimentDataHash\n    impressionFeedbackUrl\n    searchId\n    __typename\n  }\n  tileType\n  __typename\n}\n\nfragment modularModuleTileAdsPostXAd on ModularFeedTileAdsPostXAd {\n  ...modularTileAdsPostXAd\n  moduleId\n  moduleRank\n  moduleType\n  __typename\n}\n\nfragment modularTileAdsPostXAd on ModularFeedTileAdsPostXAd {\n  tileId\n  adsPostXAd {\n    ouAdId\n    adExperimentId\n    adNetwork\n    adRequestId\n    adTileType\n    clickFeedbackUrl\n    experimentDataHash\n    impressionFeedbackUrl\n    searchId\n    offer {\n      beacons {\n        noThanksClick\n        close\n        __typename\n      }\n      title\n      description\n      clickUrl\n      image\n      pixel\n      ctaYes\n      ctaNo\n      perkswallLabel\n      perkswallUrl\n      __typename\n    }\n    __typename\n  }\n  tileType\n  __typename\n}\n\nfragment modularGridModule on ModularFeedModuleGrid {\n  moduleId\n  collection\n  formFactor\n  grid {\n    actionPath\n    tiles {\n      ...modularModuleTileBingAd\n      ...modularModuleTileGoogleDisplayAd\n      ...modularModuleTileListing\n      ...modularModuleTileLocalDisplayAd\n      ...modularModuleTileSellerAd\n      __typename\n    }\n    __typename\n  }\n  moduleType\n  rank\n  rowIndex\n  searchId\n  subTitle\n  title\n  infoActionPath\n  feedIndex\n  __typename\n}\n\nfragment modularModuleTileBingAd on ModularFeedTileBingAd {\n  ...modularTileBingAd\n  moduleId\n  moduleRank\n  moduleType\n  __typename\n}\n\nfragment modularModuleTileGoogleDisplayAd on ModularFeedTileGoogleDisplayAd {\n  ...modularTileGoogleDisplayAd\n  moduleId\n  moduleRank\n  moduleType\n  __typename\n}\n\nfragment modularModuleTileListing on ModularFeedTileListing {\n  ...modularTileListing\n  moduleId\n  moduleRank\n  moduleType\n  __typename\n}\n\nfragment modularModuleTileLocalDisplayAd on ModularFeedTileLocalDisplayAd {\n  ...modularTileLocalDisplayAd\n  moduleId\n  moduleRank\n  moduleType\n  __typename\n}\n\nfragment modularModuleTileSellerAd on ModularFeedTileSellerAd {\n  ...modularTileSellerAd\n  moduleId\n  moduleRank\n  moduleType\n  __typename\n}\n\nfragment modularQueryInfo on ModularFeedQueryInfo {\n  appliedQuery\n  decisionType\n  originalQuery\n  suggestedQuery\n  __typename\n}\n
    """

    random_delay = await random_interval()
    all_listings = []
    page_cursor = None
    current_page = 0
    search_session_id = '579cfbd4-193d-44d8-8ba5-b5cefeabe928' # Get search_session_id from the website's headers or may be it's in payload

    # Loop to fetch multiple pages
    while current_page < max_pages:
        # print(f"Scrolled {count+1} pages out of {max_pages}")
        # Prepare search parameters - always include the latest page_cursor
        search_params = [
            {'key': 'platform', 'value': 'web'},
            {'key': 'lon', 'value': '-77.1995'},
            {'key': 'lat', 'value': '38.696'},
            {'key': 'zipcode', 'value': '33503'},
            {'key': 'DISTANCE', 'value': '200'}, # 200 in miles
            {'key': 'cid', 'value': str(category_id)}, # category id is based on the category of the listings, please refer to the network tools of the browser
            {'key': 'limit', 'value': '50'},
            {'key': 'searchSessionId', 'value': search_session_id}
        ]

        # Page Cursor is generate after each scroll
        if page_cursor:
            search_params.append({'key': 'page_cursor', 'value': page_cursor})

        json_data = {
            'operationName': 'GetModularFeed',
            'variables': {
                'debug': False,
                'searchParams': search_params,
            },
            'query': query,
        }

        try:
            # I have create a make_requests function sepearately, please use your own preferred requests module. I have used curl_cffi one.
            response = await make_requests(
                'post',
                'https://offerup.com/api/graphql',
                headers=headers,
                cookies=cookies,
                json=json_data,

            )

            data = response.json()

            # Check if we have valid data
            if 'data' not in data or 'modularFeed' not in data['data'] or 'looseTiles' not in data['data']['modularFeed']:
                print(f"Invalid or empty response on page {current_page + 1}")
                break

            # Get the listings from this page
            page_listings = []
            for tile in data['data']['modularFeed']['looseTiles']:
                # Check if this is a listing tile (not an ad or other content)
                if 'listing' in tile:
                    print(tile['listing']['title'])
                    listing_info = {
                        'title': tile['listing']['title'],
                        # 'id': tile['listing']['listingId'],
                        'url': f"https://offerup.com/item/detail/{tile['listing']['listingId']}",
                        'price': tile['listing']['price'],
                        # 'location': tile['listing']['locationName']
                    }
                    page_listings.append(listing_info)

            # Add these listings to our main list
            all_listings.extend(page_listings)
            await asyncio.sleep(random_delay)

            print('--------------------------------------------------')
            print(f"Page {current_page + 1}: Found {len(page_listings)} listings")
            print('--------------------------------------------------\n')

            # Get the cursor for the next page
            new_cursor = data['data']['modularFeed'].get('pageCursor')

            # If we have no new cursor or it's the same as the previous one, we're done
            if not new_cursor or new_cursor == page_cursor:
                print("No more pages available")
                break

            # Update the cursor for the next request
            page_cursor = new_cursor
            current_page += 1

        except Exception as e:
            print(f"Error fetching data: {str(e)}")
            break

    return all_listings