Browse Source

Set API by CLI

master
terorie 1 year ago
parent
commit
a9589194e7
6 changed files with 58 additions and 25 deletions
  1. 11
    3
      api/api.go
  2. 10
    1
      apiclassic/get.go
  3. 17
    18
      apiclassic/grab.go
  4. 1
    0
      apijson/grab.go
  5. 2
    2
      cmd/channeldump.go
  6. 17
    1
      main.go

+ 11
- 3
api/api.go View File

@@ -2,22 +2,30 @@ package api

import (
"github.com/terorie/yt-mango/data"
"github.com/terorie/yt-mango/classic"
"github.com/terorie/yt-mango/apiclassic"
"github.com/terorie/yt-mango/apijson"
)

type API struct {
GetVideo func(*data.Video) error
GetVideoSubtitleList func(*data.Video) error
GetChannel func(*data.Channel) error
GetChannelVideoURLs func(channelID string, page uint) ([]string, error)
}

// TODO Fallback option
var DefaultAPI *API = nil

var ClassicAPI = API{
GetVideo: apiclassic.GetVideo,
GetVideoSubtitleList: apiclassic.GetVideoSubtitleList,
GetChannel: apiclassic.GetChannel,
GetChannelVideoURLs: apiclassic.GetChannelVideoURLs,
}

var JsonAPI struct {

var JsonAPI = API{
GetVideo: apijson.GetVideo,
GetVideoSubtitleList: apiclassic.GetVideoSubtitleList,
GetChannel: apijson.GetChannel,
GetChannelVideoURLs: apijson.GetChannelVideoURLs,
}

+ 10
- 1
apiclassic/get.go View File

@@ -9,7 +9,7 @@ func GetVideo(v *data.Video) error {
if len(v.ID) == 0 { return errors.New("no video ID") }

// Download the doc tree
doc, err := grab(v)
doc, err := GrabVideo(v.ID)
if err != nil { return err }

// Parse it
@@ -20,6 +20,15 @@ func GetVideo(v *data.Video) error {
return nil
}

func GetVideoSubtitleList(v *data.Video) (err error) {
tracks, err := GrabSubtitleList(v.ID)
if err != nil { return }
for _, track := range tracks.Tracks {
v.Subtitles = append(v.Subtitles, track.LangCode)
}
return
}

func GetChannel(c *data.Channel) error {
return errors.New("not implemented")
}

+ 17
- 18
apiclassic/grab.go View File

@@ -5,7 +5,6 @@ import (
"errors"
"encoding/xml"
"github.com/PuerkitoBio/goquery"
"github.com/terorie/yt-mango/data"
"github.com/terorie/yt-mango/common"
)

@@ -13,10 +12,10 @@ const mainURL = "https://www.youtube.com/watch?has_verified=1&bpctr=6969696969&v
const subtitleURL = "https://video.google.com/timedtext?type=list&v="

// Grabs a HTML video page and returns the document tree
func grab(v *data.Video) (doc *goquery.Document, err error) {
req, err := http.NewRequest("GET", mainURL + v.ID, nil)
func GrabVideo(videoID string) (doc *goquery.Document, err error) {
req, err := http.NewRequest("GET", mainURL + videoID, nil)
if err != nil { return }
requestHeader(&req.Header)
setHeaders(&req.Header)

res, err := common.Client.Do(req)
if err != nil { return }
@@ -30,24 +29,24 @@ func grab(v *data.Video) (doc *goquery.Document, err error) {
}

// Grabs and parses a subtitle list
func grabSubtitleList(v *data.Video) (err error) {
req, err := http.NewRequest("GET", subtitleURL + v.ID, nil)
if err != nil { return err }
func GrabSubtitleList(videoID string) (tracks *XMLSubTrackList, err error) {
req, err := http.NewRequest("GET", subtitleURL + videoID, nil)
if err != nil { return }
setHeaders(&req.Header)

res, err := client.Do(req)
if err != nil { return err }
if res.StatusCode != 200 { return errors.New("HTTP failure") }
res, err := common.Client.Do(req)
if err != nil { return }
if res.StatusCode != 200 { return nil, errors.New("HTTP failure") }

defer res.Body.Close()
decoder := xml.NewDecoder(res.Body)

var tracks XMLSubTrackList
err = decoder.Decode(&tracks)
if err != nil { return err }

for _, track := range tracks.Tracks {
v.Subtitles = append(v.Subtitles, track.LangCode)
}

tracks = new(XMLSubTrackList)
err = decoder.Decode(tracks)
return
}

func setHeaders(h *http.Header) {
h.Add("Host", "www.youtube.com")
h.Add("User-Agent", "yt-mango/0.1")
}

+ 1
- 0
apijson/grab.go View File

@@ -67,6 +67,7 @@ func GrabChannelPage(channelID string, page uint) (root *fastjson.Value, err err

func setHeaders(h *http.Header) {
h.Add("Host", "www.youtube.com")
h.Add("User-Agent", "yt-mango/0.1")
h.Add("X-YouTube-Client-Name", "1")
h.Add("X-YouTube-Client-Version", "2.20170707")
}

+ 2
- 2
cmd/channeldump.go View File

@@ -9,7 +9,7 @@ import (
"time"
"bufio"
"log"
"github.com/terorie/yt-mango/apijson"
"github.com/terorie/yt-mango/api"
)

var channelDumpCmd = cobra.Command{
@@ -74,7 +74,7 @@ var channelDumpCmd = cobra.Command{

totalURLs := 0
for i := offset; true; i++ {
channelURLs, err := apijson.GetChannelVideoURLs(channelID, uint(i))
channelURLs, err := api.DefaultAPI.GetChannelVideoURLs(channelID, uint(i))
if err != nil {
log.Printf("Aborting on error %v.", err)
break

+ 17
- 1
main.go View File

@@ -9,6 +9,7 @@ import (
"os"
"github.com/terorie/yt-mango/cmd"
"log"
"github.com/terorie/yt-mango/api"
)

const Version = "v0.1 -- dev"
@@ -22,6 +23,8 @@ func main() {
log.SetOutput(os.Stderr)

var printVersion bool
var forceAPI string

rootCmd := cobra.Command{
Use: "yt-mango",
Short: "YT-Mango is a scalable video metadata archiver",
@@ -32,10 +35,23 @@ func main() {
fmt.Println(Version)
os.Exit(0)
}
switch forceAPI {
case "": break
case "classic": api.DefaultAPI = &api.ClassicAPI
case "json": api.DefaultAPI = &api.JsonAPI
default:
fmt.Fprintln(os.Stderr, "Invalid API specified.\n" +
"Valid options are: \"classic\" and \"json\"")
os.Exit(1)
}
},
}

rootCmd.Flags().BoolVar(&printVersion, "version", false,
fmt.Sprintf("Print the version (" + Version +") and exit"), )
fmt.Sprintf("Print the version (" + Version +") and exit"))
rootCmd.Flags().StringVarP(&forceAPI, "api", "a", "",
"Use the specified API for all calls.\n" +
"Possible options: \"classic\" and \"json\"")

rootCmd.AddCommand(&cmd.Channel)
rootCmd.AddCommand(&cmd.Video)

Loading…
Cancel
Save