This is a backup of https://github.com/ytdl-org/youtube-dl since it was taken down by the RIAA.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

helper.py 9.3KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246
  1. from __future__ import unicode_literals
  2. import errno
  3. import io
  4. import hashlib
  5. import json
  6. import os.path
  7. import re
  8. import types
  9. import sys
  10. import youtube_dl.extractor
  11. from youtube_dl import YoutubeDL
  12. from youtube_dl.compat import (
  13. compat_os_name,
  14. compat_str,
  15. )
  16. from youtube_dl.utils import (
  17. preferredencoding,
  18. write_string,
  19. )
  20. def get_params(override=None):
  21. PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
  22. "parameters.json")
  23. LOCAL_PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
  24. "local_parameters.json")
  25. with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
  26. parameters = json.load(pf)
  27. if os.path.exists(LOCAL_PARAMETERS_FILE):
  28. with io.open(LOCAL_PARAMETERS_FILE, encoding='utf-8') as pf:
  29. parameters.update(json.load(pf))
  30. if override:
  31. parameters.update(override)
  32. return parameters
  33. def try_rm(filename):
  34. """ Remove a file if it exists """
  35. try:
  36. os.remove(filename)
  37. except OSError as ose:
  38. if ose.errno != errno.ENOENT:
  39. raise
  40. def report_warning(message):
  41. '''
  42. Print the message to stderr, it will be prefixed with 'WARNING:'
  43. If stderr is a tty file the 'WARNING:' will be colored
  44. '''
  45. if sys.stderr.isatty() and compat_os_name != 'nt':
  46. _msg_header = '\033[0;33mWARNING:\033[0m'
  47. else:
  48. _msg_header = 'WARNING:'
  49. output = '%s %s\n' % (_msg_header, message)
  50. if 'b' in getattr(sys.stderr, 'mode', '') or sys.version_info[0] < 3:
  51. output = output.encode(preferredencoding())
  52. sys.stderr.write(output)
  53. class FakeYDL(YoutubeDL):
  54. def __init__(self, override=None):
  55. # Different instances of the downloader can't share the same dictionary
  56. # some test set the "sublang" parameter, which would break the md5 checks.
  57. params = get_params(override=override)
  58. super(FakeYDL, self).__init__(params, auto_init=False)
  59. self.result = []
  60. def to_screen(self, s, skip_eol=None):
  61. print(s)
  62. def trouble(self, s, tb=None):
  63. raise Exception(s)
  64. def download(self, x):
  65. self.result.append(x)
  66. def expect_warning(self, regex):
  67. # Silence an expected warning matching a regex
  68. old_report_warning = self.report_warning
  69. def report_warning(self, message):
  70. if re.match(regex, message):
  71. return
  72. old_report_warning(message)
  73. self.report_warning = types.MethodType(report_warning, self)
  74. def gettestcases(include_onlymatching=False):
  75. for ie in youtube_dl.extractor.gen_extractors():
  76. for tc in ie.get_testcases(include_onlymatching):
  77. yield tc
  78. md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
  79. def expect_value(self, got, expected, field):
  80. if isinstance(expected, compat_str) and expected.startswith('re:'):
  81. match_str = expected[len('re:'):]
  82. match_rex = re.compile(match_str)
  83. self.assertTrue(
  84. isinstance(got, compat_str),
  85. 'Expected a %s object, but got %s for field %s' % (
  86. compat_str.__name__, type(got).__name__, field))
  87. self.assertTrue(
  88. match_rex.match(got),
  89. 'field %s (value: %r) should match %r' % (field, got, match_str))
  90. elif isinstance(expected, compat_str) and expected.startswith('startswith:'):
  91. start_str = expected[len('startswith:'):]
  92. self.assertTrue(
  93. isinstance(got, compat_str),
  94. 'Expected a %s object, but got %s for field %s' % (
  95. compat_str.__name__, type(got).__name__, field))
  96. self.assertTrue(
  97. got.startswith(start_str),
  98. 'field %s (value: %r) should start with %r' % (field, got, start_str))
  99. elif isinstance(expected, compat_str) and expected.startswith('contains:'):
  100. contains_str = expected[len('contains:'):]
  101. self.assertTrue(
  102. isinstance(got, compat_str),
  103. 'Expected a %s object, but got %s for field %s' % (
  104. compat_str.__name__, type(got).__name__, field))
  105. self.assertTrue(
  106. contains_str in got,
  107. 'field %s (value: %r) should contain %r' % (field, got, contains_str))
  108. elif isinstance(expected, type):
  109. self.assertTrue(
  110. isinstance(got, expected),
  111. 'Expected type %r for field %s, but got value %r of type %r' % (expected, field, got, type(got)))
  112. elif isinstance(expected, dict) and isinstance(got, dict):
  113. expect_dict(self, got, expected)
  114. elif isinstance(expected, list) and isinstance(got, list):
  115. self.assertEqual(
  116. len(expected), len(got),
  117. 'Expect a list of length %d, but got a list of length %d for field %s' % (
  118. len(expected), len(got), field))
  119. for index, (item_got, item_expected) in enumerate(zip(got, expected)):
  120. type_got = type(item_got)
  121. type_expected = type(item_expected)
  122. self.assertEqual(
  123. type_expected, type_got,
  124. 'Type mismatch for list item at index %d for field %s, expected %r, got %r' % (
  125. index, field, type_expected, type_got))
  126. expect_value(self, item_got, item_expected, field)
  127. else:
  128. if isinstance(expected, compat_str) and expected.startswith('md5:'):
  129. self.assertTrue(
  130. isinstance(got, compat_str),
  131. 'Expected field %s to be a unicode object, but got value %r of type %r' % (field, got, type(got)))
  132. got = 'md5:' + md5(got)
  133. elif isinstance(expected, compat_str) and expected.startswith('mincount:'):
  134. self.assertTrue(
  135. isinstance(got, (list, dict)),
  136. 'Expected field %s to be a list or a dict, but it is of type %s' % (
  137. field, type(got).__name__))
  138. expected_num = int(expected.partition(':')[2])
  139. assertGreaterEqual(
  140. self, len(got), expected_num,
  141. 'Expected %d items in field %s, but only got %d' % (expected_num, field, len(got)))
  142. return
  143. self.assertEqual(
  144. expected, got,
  145. 'Invalid value for field %s, expected %r, got %r' % (field, expected, got))
  146. def expect_dict(self, got_dict, expected_dict):
  147. for info_field, expected in expected_dict.items():
  148. got = got_dict.get(info_field)
  149. expect_value(self, got, expected, info_field)
  150. def expect_info_dict(self, got_dict, expected_dict):
  151. expect_dict(self, got_dict, expected_dict)
  152. # Check for the presence of mandatory fields
  153. if got_dict.get('_type') not in ('playlist', 'multi_video'):
  154. for key in ('id', 'url', 'title', 'ext'):
  155. self.assertTrue(got_dict.get(key), 'Missing mandatory field %s' % key)
  156. # Check for mandatory fields that are automatically set by YoutubeDL
  157. for key in ['webpage_url', 'extractor', 'extractor_key']:
  158. self.assertTrue(got_dict.get(key), 'Missing field: %s' % key)
  159. # Are checkable fields missing from the test case definition?
  160. test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value))
  161. for key, value in got_dict.items()
  162. if value and key in ('id', 'title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location', 'age_limit'))
  163. missing_keys = set(test_info_dict.keys()) - set(expected_dict.keys())
  164. if missing_keys:
  165. def _repr(v):
  166. if isinstance(v, compat_str):
  167. return "'%s'" % v.replace('\\', '\\\\').replace("'", "\\'").replace('\n', '\\n')
  168. else:
  169. return repr(v)
  170. info_dict_str = ''
  171. if len(missing_keys) != len(expected_dict):
  172. info_dict_str += ''.join(
  173. ' %s: %s,\n' % (_repr(k), _repr(v))
  174. for k, v in test_info_dict.items() if k not in missing_keys)
  175. if info_dict_str:
  176. info_dict_str += '\n'
  177. info_dict_str += ''.join(
  178. ' %s: %s,\n' % (_repr(k), _repr(test_info_dict[k]))
  179. for k in missing_keys)
  180. write_string(
  181. '\n\'info_dict\': {\n' + info_dict_str + '},\n', out=sys.stderr)
  182. self.assertFalse(
  183. missing_keys,
  184. 'Missing keys in test definition: %s' % (
  185. ', '.join(sorted(missing_keys))))
  186. def assertRegexpMatches(self, text, regexp, msg=None):
  187. if hasattr(self, 'assertRegexp'):
  188. return self.assertRegexp(text, regexp, msg)
  189. else:
  190. m = re.match(regexp, text)
  191. if not m:
  192. note = 'Regexp didn\'t match: %r not found' % (regexp)
  193. if len(text) < 1000:
  194. note += ' in %r' % text
  195. if msg is None:
  196. msg = note
  197. else:
  198. msg = note + ', ' + msg
  199. self.assertTrue(m, msg)
  200. def assertGreaterEqual(self, got, expected, msg=None):
  201. if not (got >= expected):
  202. if msg is None:
  203. msg = '%r not greater than or equal to %r' % (got, expected)
  204. self.assertTrue(got >= expected, msg)
  205. def expect_warnings(ydl, warnings_re):
  206. real_warning = ydl.report_warning
  207. def _report_warning(w):
  208. if not any(re.search(w_re, w) for w_re in warnings_re):
  209. real_warning(w)
  210. ydl.report_warning = _report_warning