You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

grab.go 1.7KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970
  1. package classic
  2. import (
  3. "net/http"
  4. "errors"
  5. "encoding/xml"
  6. "time"
  7. "github.com/PuerkitoBio/goquery"
  8. "github.com/terorie/youtube-mango/data"
  9. )
  10. var transport = http.Transport{
  11. MaxIdleConns: 10,
  12. IdleConnTimeout: 30 * time.Second,
  13. }
  14. var client = http.Client{Transport: &transport}
  15. const mainURL = "https://www.youtube.com/watch?has_verified=1&bpctr=6969696969&v="
  16. const subtitleURL = "https://video.google.com/timedtext?type=list&v="
  17. // Grabs a HTML video page and returns the document tree
  18. func grab(v *data.Video) (doc *goquery.Document, err error) {
  19. req, err := http.NewRequest("GET", mainURL + v.ID, nil)
  20. if err != nil { return }
  21. requestHeader(&req.Header)
  22. res, err := client.Do(req)
  23. if err != nil { return }
  24. if res.StatusCode != 200 { return nil, errors.New("HTTP failure") }
  25. defer res.Body.Close()
  26. doc, err = goquery.NewDocumentFromReader(res.Body)
  27. if err != nil { return nil, err }
  28. return
  29. }
  30. // Grabs and parses a subtitle list
  31. func grabSubtitleList(v *data.Video) (err error) {
  32. req, err := http.NewRequest("GET", subtitleURL + v.ID, nil)
  33. if err != nil { return err }
  34. requestHeader(&req.Header)
  35. res, err := client.Do(req)
  36. if err != nil { return err }
  37. if res.StatusCode != 200 { return errors.New("HTTP failure") }
  38. defer res.Body.Close()
  39. decoder := xml.NewDecoder(res.Body)
  40. var tracks XMLSubTrackList
  41. err = decoder.Decode(&tracks)
  42. if err != nil { return err }
  43. for _, track := range tracks.Tracks {
  44. v.Subtitles = append(v.Subtitles, track.LangCode)
  45. }
  46. return
  47. }
  48. // Important:
  49. // - Set header "Accept-Language: en-US" or else parser might break
  50. // - Set header "User-Agent: youtube-mango/1.0"
  51. func requestHeader(h *http.Header) {
  52. h.Add("Accept-Language", "en-US")
  53. h.Add("User-Agent", "youtube-mango/0.1")
  54. }