package builders import ( "fmt" "net/http" "sort" "strconv" "strings" "time" "github.com/BrianHicks/finch/duration" "github.com/pkg/errors" "google.golang.org/api/youtube/v3" "github.com/mxpv/podsync/pkg/api" "github.com/mxpv/podsync/pkg/model" ) const ( maxYoutubeResults = 50 hdBytesPerSecond = 350000 ldBytesPerSecond = 100000 lowAudioBytesPerSecond = 48000 / 8 highAudioBytesPerSecond = 128000 / 8 ) type apiKey string func (key apiKey) Get() (string, string) { return "key", string(key) } type YouTubeBuilder struct { client *youtube.Service key apiKey } // Cost: 5 units (call method: 1, snippet: 2, contentDetails: 2) // See https://developers.google.com/youtube/v3/docs/channels/list#part func (yt *YouTubeBuilder) listChannels(linkType api.LinkType, id string, parts string) (*youtube.Channel, error) { req := yt.client.Channels.List(parts) switch linkType { case api.LinkTypeChannel: req = req.Id(id) case api.LinkTypeUser: req = req.ForUsername(id) default: return nil, errors.New("unsupported link type") } resp, err := req.Do(yt.key) if err != nil { return nil, errors.Wrapf(err, "failed to query channel") } if len(resp.Items) == 0 { return nil, api.ErrNotFound } item := resp.Items[0] return item, nil } // Cost: 3 units (call method: 1, snippet: 2) // See https://developers.google.com/youtube/v3/docs/playlists/list#part func (yt *YouTubeBuilder) listPlaylists(id, channelID string, parts string) (*youtube.Playlist, error) { req := yt.client.Playlists.List(parts) if id != "" { req = req.Id(id) } else { req = req.ChannelId(channelID) } resp, err := req.Do(yt.key) if err != nil { return nil, errors.Wrapf(err, "failed to query playlist") } if len(resp.Items) == 0 { return nil, api.ErrNotFound } item := resp.Items[0] return item, nil } // Cost: 3 units (call: 1, snippet: 2) // See https://developers.google.com/youtube/v3/docs/playlistItems/list#part func (yt *YouTubeBuilder) listPlaylistItems(itemID string, pageToken string) ([]*youtube.PlaylistItem, string, error) { req := yt.client.PlaylistItems.List("id,snippet").MaxResults(maxYoutubeResults).PlaylistId(itemID) if pageToken != "" { req = req.PageToken(pageToken) } resp, err := req.Do(yt.key) if err != nil { return nil, "", errors.Wrap(err, "failed to query playlist items") } return resp.Items, resp.NextPageToken, nil } func (yt *YouTubeBuilder) parseDate(s string) (time.Time, error) { date, err := time.Parse(time.RFC3339, s) if err != nil { return time.Time{}, errors.Wrapf(err, "failed to parse date: %s", s) } return date, nil } func (yt *YouTubeBuilder) selectThumbnail(snippet *youtube.ThumbnailDetails, quality api.Quality, videoID string) string { if snippet == nil { if videoID != "" { return fmt.Sprintf("https://img.youtube.com/vi/%s/default.jpg", videoID) } // TODO: use Podsync's preview image if unable to retrieve from YouTube return "" } // Use high resolution thumbnails for high quality mode // https://github.com/mxpv/Podsync/issues/14 if quality == api.QualityHigh { if snippet.Maxres != nil { return snippet.Maxres.Url } if snippet.High != nil { return snippet.High.Url } if snippet.Medium != nil { return snippet.Medium.Url } } return snippet.Default.Url } func (yt *YouTubeBuilder) GetVideoCount(feed *model.Feed) (uint64, error) { switch feed.LinkType { case api.LinkTypeChannel, api.LinkTypeUser: // Cost: 3 units if channel, err := yt.listChannels(feed.LinkType, feed.ItemID, "id,statistics"); err != nil { return 0, err } else { return channel.Statistics.VideoCount, nil } case api.LinkTypePlaylist: // Cost: 3 units if playlist, err := yt.listPlaylists(feed.ItemID, "", "id,contentDetails"); err != nil { return 0, err } else { return uint64(playlist.ContentDetails.ItemCount), nil } default: return 0, errors.New("unsupported link format") } } func (yt *YouTubeBuilder) queryFeed(feed *model.Feed) (string, error) { var ( title string desc string link string // URL link to YouTube resource itemID string // ID of YouTube's channel, user or playlist author string pubDate time.Time thumbnails *youtube.ThumbnailDetails ) switch feed.LinkType { case api.LinkTypeChannel, api.LinkTypeUser: // Cost: 5 units for channel or user channel, err := yt.listChannels(feed.LinkType, feed.ItemID, "id,snippet,contentDetails") if err != nil { return "", err } title = channel.Snippet.Title desc = channel.Snippet.Description if channel.Kind == "youtube#channel" { link = fmt.Sprintf("https://youtube.com/channel/%s", channel.Id) author = title } else { link = fmt.Sprintf("https://youtube.com/user/%s", channel.Snippet.CustomUrl) author = channel.Snippet.CustomUrl } itemID = channel.ContentDetails.RelatedPlaylists.Uploads if date, err := yt.parseDate(channel.Snippet.PublishedAt); err != nil { return "", err } else { pubDate = date } thumbnails = channel.Snippet.Thumbnails case api.LinkTypePlaylist: // Cost: 3 units for playlist playlist, err := yt.listPlaylists(feed.ItemID, "", "id,snippet") if err != nil { return "", err } title = fmt.Sprintf("%s: %s", playlist.Snippet.ChannelTitle, playlist.Snippet.Title) desc = playlist.Snippet.Description link = fmt.Sprintf("https://youtube.com/playlist?list=%s", playlist.Id) itemID = playlist.Id author = title if date, err := yt.parseDate(playlist.Snippet.PublishedAt); err != nil { return "", err } else { pubDate = date } thumbnails = playlist.Snippet.Thumbnails default: return "", errors.New("unsupported link format") } // Apply customizations and default values if desc == "" { desc = fmt.Sprintf("%s (%s)", title, pubDate) } var image string if feed.CoverArt != "" { image = feed.CoverArt } else { image = yt.selectThumbnail(thumbnails, feed.Quality, "") } // New interface feed.Title = title feed.Description = desc feed.Author = author feed.ItemURL = link feed.UpdatedAt = time.Now().UTC() feed.PubDate = pubDate feed.CoverArt = image return itemID, nil } // Video size information requires 1 additional call for each video (1 feed = 50 videos = 50 calls), // which is too expensive, so get approximated size depending on duration and definition params func (yt *YouTubeBuilder) getSize(duration int64, feed *model.Feed) int64 { if feed.Format == api.FormatAudio { if feed.Quality == api.QualityHigh { return highAudioBytesPerSecond * duration } return lowAudioBytesPerSecond * duration } // Video format if feed.Quality == api.QualityHigh { return duration * hdBytesPerSecond } return duration * ldBytesPerSecond } // Cost: 5 units (call: 1, snippet: 2, contentDetails: 2) // See https://developers.google.com/youtube/v3/docs/videos/list#part func (yt *YouTubeBuilder) queryVideoDescriptions(playlist map[string]*youtube.PlaylistItemSnippet, feed *model.Feed) error { // Make the list of video ids ids := make([]string, 0, len(playlist)) for _, s := range playlist { ids = append(ids, s.ResourceId.VideoId) } req, err := yt.client.Videos.List("id,snippet,contentDetails").Id(strings.Join(ids, ",")).Do(yt.key) if err != nil { return errors.Wrap(err, "failed to query video descriptions") } for _, video := range req.Items { var ( snippet = video.Snippet videoID = video.Id videoURL = fmt.Sprintf("https://youtube.com/watch?v=%s", video.Id) image = yt.selectThumbnail(snippet.Thumbnails, feed.Quality, videoID) ) // Parse date added to playlist / publication date dateStr := "" playlistItem, ok := playlist[video.Id] if ok { dateStr = playlistItem.PublishedAt } else { dateStr = snippet.PublishedAt } pubDate, err := yt.parseDate(dateStr) if err != nil { return errors.Wrapf(err, "failed to parse video publish date: %s", dateStr) } // Sometimes YouTube retrun empty content defailt, use arbitrary one var seconds int64 = 1 if video.ContentDetails != nil { // Parse duration d, err := duration.FromString(video.ContentDetails.Duration) if err != nil { return errors.Wrapf(err, "failed to parse duration %s", video.ContentDetails.Duration) } seconds = int64(d.ToDuration().Seconds()) } var ( order = strconv.FormatInt(playlistItem.Position, 10) size = yt.getSize(seconds, feed) ) feed.Episodes = append(feed.Episodes, &model.Item{ ID: video.Id, Title: snippet.Title, Description: snippet.Description, Thumbnail: image, Duration: seconds, Size: size, VideoURL: videoURL, PubDate: model.Timestamp(pubDate), // Need for sorting Order: order, }) } return nil } // Cost: (3 units + 5 units) * X pages = 8 units per page func (yt *YouTubeBuilder) queryItems(itemID string, feed *model.Feed) error { var ( token string count int ) for { items, pageToken, err := yt.listPlaylistItems(itemID, token) if err != nil { return err } token = pageToken if len(items) == 0 { return nil } // Extract playlist snippets snippets := map[string]*youtube.PlaylistItemSnippet{} for _, item := range items { snippets[item.Snippet.ResourceId.VideoId] = item.Snippet count++ } // Query video descriptions from the list of ids if err := yt.queryVideoDescriptions(snippets, feed); err != nil { return err } if count >= feed.PageSize || token == "" { return nil } } } func (yt *YouTubeBuilder) Build(feed *model.Feed) error { feed.Episodes = []*model.Item{} // Query general information about feed (title, description, lang, etc) itemID, err := yt.queryFeed(feed) if err != nil { return err } // Get video descriptions if feed.PageSize == 0 { feed.PageSize = maxYoutubeResults } if err := yt.queryItems(itemID, feed); err != nil { return err } // New interface sort.Slice(feed.Episodes, func(i, j int) bool { item1, _ := strconv.Atoi(feed.Episodes[i].Order) item2, _ := strconv.Atoi(feed.Episodes[j].Order) return item1 < item2 }) if len(feed.Episodes) > 0 { feed.LastID = feed.Episodes[0].ID } else { feed.LastID = "" } return nil } func NewYouTubeBuilder(key string) (*YouTubeBuilder, error) { yt, err := youtube.New(&http.Client{}) if err != nil { return nil, errors.Wrap(err, "failed to create youtube client") } return &YouTubeBuilder{client: yt, key: apiKey(key)}, nil }