@@ -2,22 +2,30 @@ package api | |||
import ( | |||
"github.com/terorie/yt-mango/data" | |||
"github.com/terorie/yt-mango/classic" | |||
"github.com/terorie/yt-mango/apiclassic" | |||
"github.com/terorie/yt-mango/apijson" | |||
) | |||
type API struct { | |||
GetVideo func(*data.Video) error | |||
GetVideoSubtitleList func(*data.Video) error | |||
GetChannel func(*data.Channel) error | |||
GetChannelVideoURLs func(channelID string, page uint) ([]string, error) | |||
} | |||
// TODO Fallback option | |||
var DefaultAPI *API = nil | |||
var ClassicAPI = API{ | |||
GetVideo: apiclassic.GetVideo, | |||
GetVideoSubtitleList: apiclassic.GetVideoSubtitleList, | |||
GetChannel: apiclassic.GetChannel, | |||
GetChannelVideoURLs: apiclassic.GetChannelVideoURLs, | |||
} | |||
var JsonAPI struct { | |||
var JsonAPI = API{ | |||
GetVideo: apijson.GetVideo, | |||
GetVideoSubtitleList: apiclassic.GetVideoSubtitleList, | |||
GetChannel: apijson.GetChannel, | |||
GetChannelVideoURLs: apijson.GetChannelVideoURLs, | |||
} |
@@ -9,7 +9,7 @@ func GetVideo(v *data.Video) error { | |||
if len(v.ID) == 0 { return errors.New("no video ID") } | |||
// Download the doc tree | |||
doc, err := grab(v) | |||
doc, err := GrabVideo(v.ID) | |||
if err != nil { return err } | |||
// Parse it | |||
@@ -20,6 +20,15 @@ func GetVideo(v *data.Video) error { | |||
return nil | |||
} | |||
func GetVideoSubtitleList(v *data.Video) (err error) { | |||
tracks, err := GrabSubtitleList(v.ID) | |||
if err != nil { return } | |||
for _, track := range tracks.Tracks { | |||
v.Subtitles = append(v.Subtitles, track.LangCode) | |||
} | |||
return | |||
} | |||
func GetChannel(c *data.Channel) error { | |||
return errors.New("not implemented") | |||
} |
@@ -5,7 +5,6 @@ import ( | |||
"errors" | |||
"encoding/xml" | |||
"github.com/PuerkitoBio/goquery" | |||
"github.com/terorie/yt-mango/data" | |||
"github.com/terorie/yt-mango/common" | |||
) | |||
@@ -13,10 +12,10 @@ const mainURL = "https://www.youtube.com/watch?has_verified=1&bpctr=6969696969&v | |||
const subtitleURL = "https://video.google.com/timedtext?type=list&v=" | |||
// Grabs a HTML video page and returns the document tree | |||
func grab(v *data.Video) (doc *goquery.Document, err error) { | |||
req, err := http.NewRequest("GET", mainURL + v.ID, nil) | |||
func GrabVideo(videoID string) (doc *goquery.Document, err error) { | |||
req, err := http.NewRequest("GET", mainURL + videoID, nil) | |||
if err != nil { return } | |||
requestHeader(&req.Header) | |||
setHeaders(&req.Header) | |||
res, err := common.Client.Do(req) | |||
if err != nil { return } | |||
@@ -30,24 +29,24 @@ func grab(v *data.Video) (doc *goquery.Document, err error) { | |||
} | |||
// Grabs and parses a subtitle list | |||
func grabSubtitleList(v *data.Video) (err error) { | |||
req, err := http.NewRequest("GET", subtitleURL + v.ID, nil) | |||
if err != nil { return err } | |||
func GrabSubtitleList(videoID string) (tracks *XMLSubTrackList, err error) { | |||
req, err := http.NewRequest("GET", subtitleURL + videoID, nil) | |||
if err != nil { return } | |||
setHeaders(&req.Header) | |||
res, err := client.Do(req) | |||
if err != nil { return err } | |||
if res.StatusCode != 200 { return errors.New("HTTP failure") } | |||
res, err := common.Client.Do(req) | |||
if err != nil { return } | |||
if res.StatusCode != 200 { return nil, errors.New("HTTP failure") } | |||
defer res.Body.Close() | |||
decoder := xml.NewDecoder(res.Body) | |||
var tracks XMLSubTrackList | |||
err = decoder.Decode(&tracks) | |||
if err != nil { return err } | |||
for _, track := range tracks.Tracks { | |||
v.Subtitles = append(v.Subtitles, track.LangCode) | |||
} | |||
tracks = new(XMLSubTrackList) | |||
err = decoder.Decode(tracks) | |||
return | |||
} | |||
func setHeaders(h *http.Header) { | |||
h.Add("Host", "www.youtube.com") | |||
h.Add("User-Agent", "yt-mango/0.1") | |||
} |
@@ -67,6 +67,7 @@ func GrabChannelPage(channelID string, page uint) (root *fastjson.Value, err err | |||
func setHeaders(h *http.Header) { | |||
h.Add("Host", "www.youtube.com") | |||
h.Add("User-Agent", "yt-mango/0.1") | |||
h.Add("X-YouTube-Client-Name", "1") | |||
h.Add("X-YouTube-Client-Version", "2.20170707") | |||
} |
@@ -9,7 +9,7 @@ import ( | |||
"time" | |||
"bufio" | |||
"log" | |||
"github.com/terorie/yt-mango/apijson" | |||
"github.com/terorie/yt-mango/api" | |||
) | |||
var channelDumpCmd = cobra.Command{ | |||
@@ -74,7 +74,7 @@ var channelDumpCmd = cobra.Command{ | |||
totalURLs := 0 | |||
for i := offset; true; i++ { | |||
channelURLs, err := apijson.GetChannelVideoURLs(channelID, uint(i)) | |||
channelURLs, err := api.DefaultAPI.GetChannelVideoURLs(channelID, uint(i)) | |||
if err != nil { | |||
log.Printf("Aborting on error %v.", err) | |||
break |
@@ -9,6 +9,7 @@ import ( | |||
"os" | |||
"github.com/terorie/yt-mango/cmd" | |||
"log" | |||
"github.com/terorie/yt-mango/api" | |||
) | |||
const Version = "v0.1 -- dev" | |||
@@ -22,6 +23,8 @@ func main() { | |||
log.SetOutput(os.Stderr) | |||
var printVersion bool | |||
var forceAPI string | |||
rootCmd := cobra.Command{ | |||
Use: "yt-mango", | |||
Short: "YT-Mango is a scalable video metadata archiver", | |||
@@ -32,10 +35,23 @@ func main() { | |||
fmt.Println(Version) | |||
os.Exit(0) | |||
} | |||
switch forceAPI { | |||
case "": break | |||
case "classic": api.DefaultAPI = &api.ClassicAPI | |||
case "json": api.DefaultAPI = &api.JsonAPI | |||
default: | |||
fmt.Fprintln(os.Stderr, "Invalid API specified.\n" + | |||
"Valid options are: \"classic\" and \"json\"") | |||
os.Exit(1) | |||
} | |||
}, | |||
} | |||
rootCmd.Flags().BoolVar(&printVersion, "version", false, | |||
fmt.Sprintf("Print the version (" + Version +") and exit"), ) | |||
fmt.Sprintf("Print the version (" + Version +") and exit")) | |||
rootCmd.Flags().StringVarP(&forceAPI, "api", "a", "", | |||
"Use the specified API for all calls.\n" + | |||
"Possible options: \"classic\" and \"json\"") | |||
rootCmd.AddCommand(&cmd.Channel) | |||
rootCmd.AddCommand(&cmd.Video) |