mirrors
/
OpenLiveWriter
mirror of https://github.com/OpenLiveWriter/OpenLiveWriter.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #371 from willduff/issue247 

Issue #247 - Support loading more than 500 Google Blogger posts
This commit is contained in:
Will Duff 2016-02-10 08:24:46 -08:00
parent e1a350b014 49dd8a6dd3
commit 5c8bcd9206
1 changed files with 82 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
src/managed/OpenLiveWriter.BlogClient/Clients/GoogleBloggerv3Client.cs
View File

@ -37,6 +37,11 @@ namespace OpenLiveWriter.BlogClient.Clients
public static string BloggerServiceScope = BloggerService.Scope.Blogger; public static string BloggerServiceScope = BloggerService.Scope.Blogger;
public static char LabelDelimiter = ','; public static char LabelDelimiter = ',';
/// <summary>
/// Maximum number of results the Google Blogger v3 API will return in one request.
/// </summary>
public static int MaxResultsPerRequest = 500;
public static Task<UserCredential> GetOAuth2AuthorizationAsync(string blogId, CancellationToken taskCancellationToken) public static Task<UserCredential> GetOAuth2AuthorizationAsync(string blogId, CancellationToken taskCancellationToken)
{ {
// This async task will either find cached credentials in the IDataStore provided, or it will pop open a // This async task will either find cached credentials in the IDataStore provided, or it will pop open a
@ -337,8 +342,14 @@ namespace OpenLiveWriter.BlogClient.Clients
return new BlogPostKeyword[] { }; return new BlogPostKeyword[] { };
} }
private PostList ListRecentPosts(string blogId, int maxPosts, DateTime? now, PostsResource.ListRequest.StatusEnum status) private PostList ListRecentPosts(string blogId, int maxPosts, DateTime? now, PostsResource.ListRequest.StatusEnum status, PostList previousPage)
{ {
if (previousPage != null && string.IsNullOrWhiteSpace(previousPage.NextPageToken))
{
// The previous page was also the last page, so do nothing and return an empty list.
return new PostList();
}
var recentPostsRequest = GetService().Posts.List(blogId); var recentPostsRequest = GetService().Posts.List(blogId);
if (now.HasValue) if (now.HasValue)
{ {
@ -348,21 +359,46 @@ namespace OpenLiveWriter.BlogClient.Clients
recentPostsRequest.MaxResults = maxPosts; recentPostsRequest.MaxResults = maxPosts;
recentPostsRequest.OrderBy = PostsResource.ListRequest.OrderByEnum.Published; recentPostsRequest.OrderBy = PostsResource.ListRequest.OrderByEnum.Published;
recentPostsRequest.Status = status; recentPostsRequest.Status = status;
recentPostsRequest.PageToken = previousPage?.NextPageToken;
return recentPostsRequest.Execute(); return recentPostsRequest.Execute();
} }
public BlogPost[] GetRecentPosts(string blogId, int maxPosts, bool includeCategories, DateTime? now) public BlogPost[] GetRecentPosts(string blogId, int maxPosts, bool includeCategories, DateTime? now)
{ {
var draftRecentPostsList = ListRecentPosts(blogId, maxPosts, now, PostsResource.ListRequest.StatusEnum.Draft); // Blogger requires separate API calls to get drafts vs. live vs. scheduled posts. We aggregate each
var liveRecentPostsList = ListRecentPosts(blogId, maxPosts, now, PostsResource.ListRequest.StatusEnum.Live); // type of post separately.
var scheduledRecentPostsList = ListRecentPosts(blogId, maxPosts, now, PostsResource.ListRequest.StatusEnum.Scheduled); IList<Post> draftRecentPosts = new List<Post>();
IList<Post> liveRecentPosts = new List<Post>();
IList<Post> scheduledRecentPosts = new List<Post>();
IEnumerable<Post> allPosts = new List<Post>();
var draftRecentPosts = draftRecentPostsList.Items ?? new List<Post>(); // We keep around the PostList returned by each request to support pagination.
var liveRecentPosts = liveRecentPostsList.Items ?? new List<Post>(); PostList draftRecentPostsList = null;
var scheduledRecentPosts = scheduledRecentPostsList.Items ?? new List<Post>(); PostList liveRecentPostsList = null;
PostList scheduledRecentPostsList = null;
var allPosts = draftRecentPosts.Concat(liveRecentPosts).Concat(scheduledRecentPosts); // Google has a per-request results limit on their API.
var maxResultsPerRequest = Math.Min(maxPosts, MaxResultsPerRequest);
// We break out of the following loop depending on which one of these two cases we hit:
// (a) the number of all blog posts ever posted to this blog is greater than maxPosts, so eventually
// allPosts.count() will exceed maxPosts and we can stop making requests.
// (b) the number of all blog posts ever posted to this blog is less than maxPosts, so eventually our
// calls to ListRecentPosts() will return 0 results and we need to stop making requests.
do
{
draftRecentPostsList = ListRecentPosts(blogId, maxResultsPerRequest, now, PostsResource.ListRequest.StatusEnum.Draft, draftRecentPostsList);
liveRecentPostsList = ListRecentPosts(blogId, maxResultsPerRequest, now, PostsResource.ListRequest.StatusEnum.Live, liveRecentPostsList);
scheduledRecentPostsList = ListRecentPosts(blogId, maxResultsPerRequest, now, PostsResource.ListRequest.StatusEnum.Scheduled, scheduledRecentPostsList);
draftRecentPosts = draftRecentPostsList?.Items ?? new List<Post>();
liveRecentPosts = liveRecentPostsList?.Items ?? new List<Post>();
scheduledRecentPosts = scheduledRecentPostsList?.Items ?? new List<Post>();
allPosts = allPosts.Concat(draftRecentPosts).Concat(liveRecentPosts).Concat(scheduledRecentPosts);
} while (allPosts.Count() < maxPosts && (draftRecentPosts.Count > 0 || liveRecentPosts.Count > 0 || scheduledRecentPosts.Count > 0));
return allPosts return allPosts
.OrderByDescending(p => p.Published) .OrderByDescending(p => p.Published)
.Take(maxPosts) .Take(maxPosts)
@ -430,22 +466,53 @@ namespace OpenLiveWriter.BlogClient.Clients
return ConvertToBlogPost(getPageRequest.Execute()); return ConvertToBlogPost(getPageRequest.Execute());
} }
private PageList ListPages(string blogId, int? maxPages, PagesResource.ListRequest.StatusEnum status) private PageList ListPages(string blogId, int? maxPages, PagesResource.ListRequest.StatusEnum status, PageList previousPage)
{ {
if (previousPage != null && string.IsNullOrWhiteSpace(previousPage.NextPageToken))
{
// The previous page was also the last page, so do nothing and return an empty list.
return new PageList();
}
var getPagesRequest = GetService().Pages.List(blogId); var getPagesRequest = GetService().Pages.List(blogId);
getPagesRequest.MaxResults = maxPages; if (maxPages.HasValue)
{
// Google has a per-request results limit on their API.
getPagesRequest.MaxResults = Math.Min(maxPages.Value, MaxResultsPerRequest);
}
getPagesRequest.Status = status; getPagesRequest.Status = status;
return getPagesRequest.Execute(); return getPagesRequest.Execute();
} }
private IEnumerable<Page> ListAllPages(string blogId, int? maxPages) private IEnumerable<Page> ListAllPages(string blogId, int? maxPages)
{ {
var draftPageList = ListPages(blogId, maxPages, PagesResource.ListRequest.StatusEnum.Draft); // Blogger requires separate API calls to get drafts vs. live vs. scheduled posts. We aggregate each
var livePageList = ListPages(blogId, maxPages, PagesResource.ListRequest.StatusEnum.Live); // type of post separately.
IList<Page> draftPages = new List<Page>();
IList<Page> livePages = new List<Page>();
IEnumerable<Page> allPages = new List<Page>();
var draftPages = draftPageList.Items ?? new List<Page>(); // We keep around the PageList returned by each request to support pagination.
var livePages = livePageList.Items ?? new List<Page>(); PageList draftPagesList = null;
return draftPages.Concat(livePages); PageList livePagesList = null;
// We break out of the following loop depending on which one of these two cases we hit:
// (a) the number of all blog pages ever posted to this blog is greater than maxPages, so eventually
// allPages.count() will exceed maxPages and we can stop making requests.
// (b) the number of all blog pages ever posted to this blog is less than maxPages, so eventually our
// calls to ListPages() will return 0 results and we need to stop making requests.
do
{
draftPagesList = ListPages(blogId, maxPages, PagesResource.ListRequest.StatusEnum.Draft, draftPagesList);
livePagesList = ListPages(blogId, maxPages, PagesResource.ListRequest.StatusEnum.Live, livePagesList);
draftPages = draftPagesList?.Items ?? new List<Page>();
livePages = livePagesList?.Items ?? new List<Page>();
allPages = allPages.Concat(draftPages).Concat(livePages);
} while (allPages.Count() < maxPages && (draftPages.Count > 0 || livePages.Count > 0));
return allPages;
} }
public PageInfo[] GetPageList(string blogId) public PageInfo[] GetPageList(string blogId)