From 5f41d87a8bfd0c6fd0b2d8eb17fd6ecabb16f2ea Mon Sep 17 00:00:00 2001 From: Th0masL Date: Mon, 13 Jun 2022 01:43:56 +0300 Subject: [PATCH 1/6] Fix page_size limited at 50 on YouTube --- pkg/builder/youtube.go | 141 +++++++++++++++++++++++++------------ services/update/updater.go | 4 ++ 2 files changed, 99 insertions(+), 46 deletions(-) diff --git a/pkg/builder/youtube.go b/pkg/builder/youtube.go index 6cc9b0e..585f12e 100644 --- a/pkg/builder/youtube.go +++ b/pkg/builder/youtube.go @@ -9,6 +9,7 @@ import ( "strings" "time" + log "github.com/sirupsen/logrus" "github.com/BrianHicks/finch/duration" "github.com/mxpv/podsync/pkg/feed" "github.com/pkg/errors" @@ -264,67 +265,115 @@ func (yt *YouTubeBuilder) getSize(duration int64, feed *model.Feed) int64 { // See https://developers.google.com/youtube/v3/docs/videos/list#part func (yt *YouTubeBuilder) queryVideoDescriptions(ctx context.Context, playlist map[string]*youtube.PlaylistItemSnippet, feed *model.Feed) error { // Make the list of video ids + // and count how many API calls will be required ids := make([]string, 0, len(playlist)) + count := 0 + count_expected_api_calls := 1 for _, s := range playlist { + count++ ids = append(ids, s.ResourceId.VideoId) + // Increment the counter of expected API calls + if count == maxYoutubeResults { + count_expected_api_calls++ + count = 0 + } } - req, err := yt.client.Videos.List("id,snippet,contentDetails").Id(strings.Join(ids, ",")).Context(ctx).Do(yt.key) - if err != nil { - return errors.Wrap(err, "failed to query video descriptions") - } + log.Debugf("Expected to make %d API calls to get the descriptions for %d episode(s).", count_expected_api_calls, len(ids)) - for _, video := range req.Items { - var ( - snippet = video.Snippet - videoID = video.Id - videoURL = fmt.Sprintf("https://youtube.com/watch?v=%s", video.Id) - image = yt.selectThumbnail(snippet.Thumbnails, feed.Quality, videoID) - ) + // Init a list that will contains the aggregated strings of videos IDs (capped at 50 IDs per API Calls) + ids_list := make([]string, 0, count_expected_api_calls ) - // Parse date added to playlist / publication date - dateStr := "" - playlistItem, ok := playlist[video.Id] - if ok { - dateStr = playlistItem.PublishedAt - } else { - dateStr = snippet.PublishedAt + // Init some vars for the logic of breaking the IDs down into groups of 50 + total_count_id := 0 + count_id := 0 + temp_ids_list := make([]string, 0) + + for _, id := range ids { + total_count_id++ + count_id++ + + // If we have not yet reached the limit of the YouTube API, + // append this video ID to the temporary list + if count_id <= maxYoutubeResults { + temp_ids_list = append(temp_ids_list, id) } - pubDate, err := yt.parseDate(dateStr) + // If we have reached the limit of YouTube API, + // convert the temporary ID list into a string and + // save it into the final ID list + if count_id == maxYoutubeResults { + count_id = 0 + ids_list = append(ids_list, strings.Join(temp_ids_list, ",")) + // Reset the value of the temporary ID list + temp_ids_list = nil + } else if total_count_id == len(playlist) { + // Convert the temporary ID list into a string and append it to the final ID list + ids_list = append(ids_list, strings.Join(temp_ids_list, ",")) + // Reset the value of the temporary ID list + temp_ids_list = nil + } + } + + // Loop in each list of 50 (or less) IDs and query the description + for list_number := 0; list_number < len(ids_list); list_number++ { + req, err := yt.client.Videos.List("id,snippet,contentDetails").Id(ids_list[list_number]).Context(ctx).Do(yt.key) if err != nil { - return errors.Wrapf(err, "failed to parse video publish date: %s", dateStr) + return errors.Wrap(err, "failed to query video descriptions") } - // Sometimes YouTube retrun empty content defailt, use arbitrary one - var seconds int64 = 1 - if video.ContentDetails != nil { - // Parse duration - d, err := duration.FromString(video.ContentDetails.Duration) - if err != nil { - return errors.Wrapf(err, "failed to parse duration %s", video.ContentDetails.Duration) + for _, video := range req.Items { + var ( + snippet = video.Snippet + videoID = video.Id + videoURL = fmt.Sprintf("https://youtube.com/watch?v=%s", video.Id) + image = yt.selectThumbnail(snippet.Thumbnails, feed.Quality, videoID) + ) + + // Parse date added to playlist / publication date + dateStr := "" + playlistItem, ok := playlist[video.Id] + if ok { + dateStr = playlistItem.PublishedAt + } else { + dateStr = snippet.PublishedAt } - seconds = int64(d.ToDuration().Seconds()) + pubDate, err := yt.parseDate(dateStr) + if err != nil { + return errors.Wrapf(err, "failed to parse video publish date: %s", dateStr) + } + + // Sometimes YouTube retrun empty content defailt, use arbitrary one + var seconds int64 = 1 + if video.ContentDetails != nil { + // Parse duration + d, err := duration.FromString(video.ContentDetails.Duration) + if err != nil { + return errors.Wrapf(err, "failed to parse duration %s", video.ContentDetails.Duration) + } + + seconds = int64(d.ToDuration().Seconds()) + } + + var ( + order = strconv.FormatInt(playlistItem.Position, 10) + size = yt.getSize(seconds, feed) + ) + + feed.Episodes = append(feed.Episodes, &model.Episode{ + ID: video.Id, + Title: snippet.Title, + Description: snippet.Description, + Thumbnail: image, + Duration: seconds, + Size: size, + VideoURL: videoURL, + PubDate: pubDate, + Order: order, + Status: model.EpisodeNew, + }) } - - var ( - order = strconv.FormatInt(playlistItem.Position, 10) - size = yt.getSize(seconds, feed) - ) - - feed.Episodes = append(feed.Episodes, &model.Episode{ - ID: video.Id, - Title: snippet.Title, - Description: snippet.Description, - Thumbnail: image, - Duration: seconds, - Size: size, - VideoURL: videoURL, - PubDate: pubDate, - Order: order, - Status: model.EpisodeNew, - }) } return nil diff --git a/services/update/updater.go b/services/update/updater.go index df517c8..9121932 100644 --- a/services/update/updater.go +++ b/services/update/updater.go @@ -157,8 +157,12 @@ func (u *Manager) downloadEpisodes(ctx context.Context, feedConfig *feed.Config) // Build the list of files to download if err := u.db.WalkEpisodes(ctx, feedID, func(episode *model.Episode) error { + var ( + logger = log.WithFields(log.Fields{"episode_id": episode.ID}) + ) if episode.Status != model.EpisodeNew && episode.Status != model.EpisodeError { // File already downloaded + logger.Infof("skipping due to file already on disk") return nil } From ec5e4445cc736da7a7f3b5f986aa1eca332feb34 Mon Sep 17 00:00:00 2001 From: Th0masL Date: Mon, 13 Jun 2022 02:15:23 +0300 Subject: [PATCH 2/6] Change already downloaded message --- services/update/updater.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/update/updater.go b/services/update/updater.go index 9121932..7eac44f 100644 --- a/services/update/updater.go +++ b/services/update/updater.go @@ -162,7 +162,7 @@ func (u *Manager) downloadEpisodes(ctx context.Context, feedConfig *feed.Config) ) if episode.Status != model.EpisodeNew && episode.Status != model.EpisodeError { // File already downloaded - logger.Infof("skipping due to file already on disk") + logger.Infof("skipping due to already downloaded") return nil } From 937145a68f5f83d0807910e216de07d42654186e Mon Sep 17 00:00:00 2001 From: Th0masL Date: Mon, 13 Jun 2022 11:44:51 +0300 Subject: [PATCH 3/6] Small improvements --- pkg/builder/youtube.go | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/pkg/builder/youtube.go b/pkg/builder/youtube.go index 585f12e..f2a858d 100644 --- a/pkg/builder/youtube.go +++ b/pkg/builder/youtube.go @@ -3,6 +3,7 @@ package builder import ( "context" "fmt" + "math" "net/http" "sort" "strconv" @@ -265,20 +266,14 @@ func (yt *YouTubeBuilder) getSize(duration int64, feed *model.Feed) int64 { // See https://developers.google.com/youtube/v3/docs/videos/list#part func (yt *YouTubeBuilder) queryVideoDescriptions(ctx context.Context, playlist map[string]*youtube.PlaylistItemSnippet, feed *model.Feed) error { // Make the list of video ids - // and count how many API calls will be required ids := make([]string, 0, len(playlist)) - count := 0 - count_expected_api_calls := 1 for _, s := range playlist { - count++ ids = append(ids, s.ResourceId.VideoId) - // Increment the counter of expected API calls - if count == maxYoutubeResults { - count_expected_api_calls++ - count = 0 - } } + // Count how many API calls will be required + count_expected_api_calls := int(math.Ceil(float64(len(playlist)) / maxYoutubeResults)) + log.Debugf("Expected to make %d API calls to get the descriptions for %d episode(s).", count_expected_api_calls, len(ids)) // Init a list that will contains the aggregated strings of videos IDs (capped at 50 IDs per API Calls) @@ -316,8 +311,8 @@ func (yt *YouTubeBuilder) queryVideoDescriptions(ctx context.Context, playlist m } // Loop in each list of 50 (or less) IDs and query the description - for list_number := 0; list_number < len(ids_list); list_number++ { - req, err := yt.client.Videos.List("id,snippet,contentDetails").Id(ids_list[list_number]).Context(ctx).Do(yt.key) + for _, idsI := range ids_list { + req, err := yt.client.Videos.List("id,snippet,contentDetails").Id(idsI).Context(ctx).Do(yt.key) if err != nil { return errors.Wrap(err, "failed to query video descriptions") } From a2e788f861369abec9cd5e3b8997dabf46398484 Mon Sep 17 00:00:00 2001 From: Th0masL Date: Mon, 13 Jun 2022 12:04:02 +0300 Subject: [PATCH 4/6] Improve the slices chunking --- pkg/builder/youtube.go | 37 ++++++++----------------------------- 1 file changed, 8 insertions(+), 29 deletions(-) diff --git a/pkg/builder/youtube.go b/pkg/builder/youtube.go index f2a858d..dd0f29b 100644 --- a/pkg/builder/youtube.go +++ b/pkg/builder/youtube.go @@ -279,38 +279,17 @@ func (yt *YouTubeBuilder) queryVideoDescriptions(ctx context.Context, playlist m // Init a list that will contains the aggregated strings of videos IDs (capped at 50 IDs per API Calls) ids_list := make([]string, 0, count_expected_api_calls ) - // Init some vars for the logic of breaking the IDs down into groups of 50 - total_count_id := 0 - count_id := 0 - temp_ids_list := make([]string, 0) - - for _, id := range ids { - total_count_id++ - count_id++ - - // If we have not yet reached the limit of the YouTube API, - // append this video ID to the temporary list - if count_id <= maxYoutubeResults { - temp_ids_list = append(temp_ids_list, id) - } - - // If we have reached the limit of YouTube API, - // convert the temporary ID list into a string and - // save it into the final ID list - if count_id == maxYoutubeResults { - count_id = 0 - ids_list = append(ids_list, strings.Join(temp_ids_list, ",")) - // Reset the value of the temporary ID list - temp_ids_list = nil - } else if total_count_id == len(playlist) { - // Convert the temporary ID list into a string and append it to the final ID list - ids_list = append(ids_list, strings.Join(temp_ids_list, ",")) - // Reset the value of the temporary ID list - temp_ids_list = nil + // Chunk the list of IDs by slices limited to maxYoutubeResults + for i := 0; i < len(ids); i += maxYoutubeResults { + end := i + maxYoutubeResults + if end > len(ids) { + end = len(ids) } + // Save each slice as comma-delimited string + ids_list = append(ids_list, strings.Join(ids[i:end], ",")) } - // Loop in each list of 50 (or less) IDs and query the description + // Loop in each slices of 50 (or less) IDs and query their description for _, idsI := range ids_list { req, err := yt.client.Videos.List("id,snippet,contentDetails").Id(idsI).Context(ctx).Do(yt.key) if err != nil { From 531ceb78c390e53c3d3fc1c9be5b82aba9ec90ae Mon Sep 17 00:00:00 2001 From: Th0masL Date: Mon, 13 Jun 2022 23:56:14 +0300 Subject: [PATCH 5/6] Small improvement and fix errors reported by golangci-lint --- pkg/builder/youtube.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/builder/youtube.go b/pkg/builder/youtube.go index dd0f29b..4e2acfe 100644 --- a/pkg/builder/youtube.go +++ b/pkg/builder/youtube.go @@ -10,10 +10,10 @@ import ( "strings" "time" - log "github.com/sirupsen/logrus" "github.com/BrianHicks/finch/duration" "github.com/mxpv/podsync/pkg/feed" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" "google.golang.org/api/youtube/v3" "github.com/mxpv/podsync/pkg/model" @@ -272,12 +272,12 @@ func (yt *YouTubeBuilder) queryVideoDescriptions(ctx context.Context, playlist m } // Count how many API calls will be required - count_expected_api_calls := int(math.Ceil(float64(len(playlist)) / maxYoutubeResults)) + countExpectedAPICalls := int(math.Ceil(float64(len(playlist)) / maxYoutubeResults)) - log.Debugf("Expected to make %d API calls to get the descriptions for %d episode(s).", count_expected_api_calls, len(ids)) + log.Debugf("Expected to make %d API calls to get the descriptions for %d episode(s).", countExpectedAPICalls, len(ids)) // Init a list that will contains the aggregated strings of videos IDs (capped at 50 IDs per API Calls) - ids_list := make([]string, 0, count_expected_api_calls ) + ids_list := make([]string, 0, 1) // Chunk the list of IDs by slices limited to maxYoutubeResults for i := 0; i < len(ids); i += maxYoutubeResults { From 906a5a6216af256ee48694c1ececf30111b902c6 Mon Sep 17 00:00:00 2001 From: Th0masL Date: Tue, 14 Jun 2022 02:38:03 +0300 Subject: [PATCH 6/6] Some more improvements --- pkg/builder/youtube.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/pkg/builder/youtube.go b/pkg/builder/youtube.go index 4e2acfe..fc02b3c 100644 --- a/pkg/builder/youtube.go +++ b/pkg/builder/youtube.go @@ -3,7 +3,6 @@ package builder import ( "context" "fmt" - "math" "net/http" "sort" "strconv" @@ -271,13 +270,8 @@ func (yt *YouTubeBuilder) queryVideoDescriptions(ctx context.Context, playlist m ids = append(ids, s.ResourceId.VideoId) } - // Count how many API calls will be required - countExpectedAPICalls := int(math.Ceil(float64(len(playlist)) / maxYoutubeResults)) - - log.Debugf("Expected to make %d API calls to get the descriptions for %d episode(s).", countExpectedAPICalls, len(ids)) - // Init a list that will contains the aggregated strings of videos IDs (capped at 50 IDs per API Calls) - ids_list := make([]string, 0, 1) + idsList := make([]string, 0, 1) // Chunk the list of IDs by slices limited to maxYoutubeResults for i := 0; i < len(ids); i += maxYoutubeResults { @@ -286,11 +280,14 @@ func (yt *YouTubeBuilder) queryVideoDescriptions(ctx context.Context, playlist m end = len(ids) } // Save each slice as comma-delimited string - ids_list = append(ids_list, strings.Join(ids[i:end], ",")) + idsList = append(idsList, strings.Join(ids[i:end], ",")) } + // Show how many API calls will be required + log.Debugf("Expected to make %d API calls to get the descriptions for %d episode(s).", len(idsList), len(ids)) + // Loop in each slices of 50 (or less) IDs and query their description - for _, idsI := range ids_list { + for _, idsI := range idsList { req, err := yt.client.Videos.List("id,snippet,contentDetails").Id(idsI).Context(ctx).Do(yt.key) if err != nil { return errors.Wrap(err, "failed to query video descriptions")