Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- public static List<Label> GetLabels()
- {
- /*
- * use linq's lazy loading feature to enumerate an unlimited number of page requests. so i created AggregateUntil() to
- * take in a Func() which will inform when to stop iterating - in this case, we want to stop after the entries count
- * exceeds the totalNumEntries value, which Google Adwords returns from the first query
- */
- var labels = Enumerable
- .Range(0, Int32.MaxValue) //range generates a list of integers, from 1 to practically infinity
- .AggregateUntil(
- new LabelPage()
- {
- entries = new Label[] { },
- totalNumEntries = 0,
- },
- (accumulatedLabelPage, pageIndex) =>
- {
- /*
- * also use select() and first() to mitigate rate limiting; setup 5 identical requests and while the
- * returned page is null (meaning there was a caught exception), keep retrying. after first() stops
- * the evaluation, or you've attempted up to the max number of times to fetch, there will be one
- * non-empty page result. if there is no result at all, Single() will throw an error, which is what we
- * want to happen
- */
- var thisLabelsPage = Enumerable
- .Range(0, pageMaxFetchAttempts)
- .Select(pageAttempt =>
- {
- // this is the exponential backoff
- Thread.Sleep(2500 * pageAttempt * pageAttempt);
- try
- {
- var labelsAwql =
- $"select LabelId,LabelName,LabelStatus limit {(pageIndex * pageSize)},{pageSize}";
- var attemptedlabelsPage = Config.SupplyLabelService.query(labelsAwql);
- return attemptedlabelsPage;
- }
- catch (Exception e)
- {
- return new LabelPage();
- }
- })
- .First(page => page != null);
- return new LabelPage()
- {
- entries = accumulatedLabelPage
- .entries
- .ToList()
- .Concat(thisLabelsPage.entries)
- .ToArray(),
- totalNumEntries = thisLabelsPage.totalNumEntries,
- totalNumEntriesSpecified = thisLabelsPage.totalNumEntriesSpecified,
- PageType = thisLabelsPage.PageType,
- };
- },
- (accumulatedLabelPage) =>
- {
- var shouldStop = accumulatedLabelPage.entries.Length >= accumulatedLabelPage.totalNumEntries;
- return shouldStop;
- })
- .entries
- .ToList();
- return labels;
- }
Add Comment
Please, Sign In to add comment