test_x264.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485
  1. #!/usr/bin/env python
  2. import operator
  3. from optparse import OptionGroup
  4. import sys
  5. from time import time
  6. from digress.cli import Dispatcher as _Dispatcher
  7. from digress.errors import ComparisonError, FailedTestError, DisabledTestError
  8. from digress.testing import depends, comparer, Fixture, Case
  9. from digress.comparers import compare_pass
  10. from digress.scm import git as x264git
  11. from subprocess import Popen, PIPE, STDOUT
  12. import os
  13. import re
  14. import shlex
  15. import inspect
  16. from random import randrange, seed
  17. from math import ceil
  18. from itertools import imap, izip
  19. os.chdir(os.path.join(os.path.dirname(__file__), ".."))
  20. # options
  21. OPTIONS = [
  22. [ "--tune %s" % t for t in ("film", "zerolatency") ],
  23. ("", "--intra-refresh"),
  24. ("", "--no-cabac"),
  25. ("", "--interlaced"),
  26. ("", "--slice-max-size 1000"),
  27. ("", "--frame-packing 5"),
  28. [ "--preset %s" % p for p in ("ultrafast",
  29. "superfast",
  30. "veryfast",
  31. "faster",
  32. "fast",
  33. "medium",
  34. "slow",
  35. "slower",
  36. "veryslow",
  37. "placebo") ]
  38. ]
  39. # end options
  40. def compare_yuv_output(width, height):
  41. def _compare_yuv_output(file_a, file_b):
  42. size_a = os.path.getsize(file_a)
  43. size_b = os.path.getsize(file_b)
  44. if size_a != size_b:
  45. raise ComparisonError("%s is not the same size as %s" % (
  46. file_a,
  47. file_b
  48. ))
  49. BUFFER_SIZE = 8196
  50. offset = 0
  51. with open(file_a) as f_a:
  52. with open(file_b) as f_b:
  53. for chunk_a, chunk_b in izip(
  54. imap(
  55. lambda i: f_a.read(BUFFER_SIZE),
  56. xrange(size_a // BUFFER_SIZE + 1)
  57. ),
  58. imap(
  59. lambda i: f_b.read(BUFFER_SIZE),
  60. xrange(size_b // BUFFER_SIZE + 1)
  61. )
  62. ):
  63. chunk_size = len(chunk_a)
  64. if chunk_a != chunk_b:
  65. for i in xrange(chunk_size):
  66. if chunk_a[i] != chunk_b[i]:
  67. # calculate the macroblock, plane and frame from the offset
  68. offs = offset + i
  69. y_plane_area = width * height
  70. u_plane_area = y_plane_area + y_plane_area * 0.25
  71. v_plane_area = u_plane_area + y_plane_area * 0.25
  72. pixel = offs % v_plane_area
  73. frame = offs // v_plane_area
  74. if pixel < y_plane_area:
  75. plane = "Y"
  76. pixel_x = pixel % width
  77. pixel_y = pixel // width
  78. macroblock = (ceil(pixel_x / 16.0), ceil(pixel_y / 16.0))
  79. elif pixel < u_plane_area:
  80. plane = "U"
  81. pixel -= y_plane_area
  82. pixel_x = pixel % width
  83. pixel_y = pixel // width
  84. macroblock = (ceil(pixel_x / 8.0), ceil(pixel_y / 8.0))
  85. else:
  86. plane = "V"
  87. pixel -= u_plane_area
  88. pixel_x = pixel % width
  89. pixel_y = pixel // width
  90. macroblock = (ceil(pixel_x / 8.0), ceil(pixel_y / 8.0))
  91. macroblock = tuple([ int(x) for x in macroblock ])
  92. raise ComparisonError("%s differs from %s at frame %d, " \
  93. "macroblock %s on the %s plane (offset %d)" % (
  94. file_a,
  95. file_b,
  96. frame,
  97. macroblock,
  98. plane,
  99. offs)
  100. )
  101. offset += chunk_size
  102. return _compare_yuv_output
  103. def program_exists(program):
  104. def is_exe(fpath):
  105. return os.path.exists(fpath) and os.access(fpath, os.X_OK)
  106. fpath, fname = os.path.split(program)
  107. if fpath:
  108. if is_exe(program):
  109. return program
  110. else:
  111. for path in os.environ["PATH"].split(os.pathsep):
  112. exe_file = os.path.join(path, program)
  113. if is_exe(exe_file):
  114. return exe_file
  115. return None
  116. class x264(Fixture):
  117. scm = x264git
  118. class Compile(Case):
  119. @comparer(compare_pass)
  120. def test_configure(self):
  121. Popen([
  122. "make",
  123. "distclean"
  124. ], stdout=PIPE, stderr=STDOUT).communicate()
  125. configure_proc = Popen([
  126. "./configure"
  127. ] + self.fixture.dispatcher.configure, stdout=PIPE, stderr=STDOUT)
  128. output = configure_proc.communicate()[0]
  129. if configure_proc.returncode != 0:
  130. raise FailedTestError("configure failed: %s" % output.replace("\n", " "))
  131. @depends("configure")
  132. @comparer(compare_pass)
  133. def test_make(self):
  134. make_proc = Popen([
  135. "make",
  136. "-j5"
  137. ], stdout=PIPE, stderr=STDOUT)
  138. output = make_proc.communicate()[0]
  139. if make_proc.returncode != 0:
  140. raise FailedTestError("make failed: %s" % output.replace("\n", " "))
  141. _dimension_pattern = re.compile(r"\w+ [[]info[]]: (\d+)x(\d+)[pi] \d+:\d+ @ \d+/\d+ fps [(][vc]fr[)]")
  142. def _YUVOutputComparisonFactory():
  143. class YUVOutputComparison(Case):
  144. _dimension_pattern = _dimension_pattern
  145. depends = [ Compile ]
  146. options = []
  147. def __init__(self):
  148. for name, meth in inspect.getmembers(self):
  149. if name[:5] == "test_" and name[5:] not in self.fixture.dispatcher.yuv_tests:
  150. delattr(self.__class__, name)
  151. def _run_x264(self):
  152. x264_proc = Popen([
  153. "./x264",
  154. "-o",
  155. "%s.264" % self.fixture.dispatcher.video,
  156. "--dump-yuv",
  157. "x264-output.yuv"
  158. ] + self.options + [
  159. self.fixture.dispatcher.video
  160. ], stdout=PIPE, stderr=STDOUT)
  161. output = x264_proc.communicate()[0]
  162. if x264_proc.returncode != 0:
  163. raise FailedTestError("x264 did not complete properly: %s" % output.replace("\n", " "))
  164. matches = _dimension_pattern.match(output)
  165. return (int(matches.group(1)), int(matches.group(2)))
  166. @comparer(compare_pass)
  167. def test_jm(self):
  168. if not program_exists("ldecod"): raise DisabledTestError("jm unavailable")
  169. try:
  170. runres = self._run_x264()
  171. jm_proc = Popen([
  172. "ldecod",
  173. "-i",
  174. "%s.264" % self.fixture.dispatcher.video,
  175. "-o",
  176. "jm-output.yuv"
  177. ], stdout=PIPE, stderr=STDOUT)
  178. output = jm_proc.communicate()[0]
  179. if jm_proc.returncode != 0:
  180. raise FailedTestError("jm did not complete properly: %s" % output.replace("\n", " "))
  181. try:
  182. compare_yuv_output(*runres)("x264-output.yuv", "jm-output.yuv")
  183. except ComparisonError, e:
  184. raise FailedTestError(e)
  185. finally:
  186. try: os.remove("x264-output.yuv")
  187. except: pass
  188. try: os.remove("%s.264" % self.fixture.dispatcher.video)
  189. except: pass
  190. try: os.remove("jm-output.yuv")
  191. except: pass
  192. try: os.remove("log.dec")
  193. except: pass
  194. try: os.remove("dataDec.txt")
  195. except: pass
  196. @comparer(compare_pass)
  197. def test_ffmpeg(self):
  198. if not program_exists("ffmpeg"): raise DisabledTestError("ffmpeg unavailable")
  199. try:
  200. runres = self._run_x264()
  201. ffmpeg_proc = Popen([
  202. "ffmpeg",
  203. "-vsync 0",
  204. "-i",
  205. "%s.264" % self.fixture.dispatcher.video,
  206. "ffmpeg-output.yuv"
  207. ], stdout=PIPE, stderr=STDOUT)
  208. output = ffmpeg_proc.communicate()[0]
  209. if ffmpeg_proc.returncode != 0:
  210. raise FailedTestError("ffmpeg did not complete properly: %s" % output.replace("\n", " "))
  211. try:
  212. compare_yuv_output(*runres)("x264-output.yuv", "ffmpeg-output.yuv")
  213. except ComparisonError, e:
  214. raise FailedTestError(e)
  215. finally:
  216. try: os.remove("x264-output.yuv")
  217. except: pass
  218. try: os.remove("%s.264" % self.fixture.dispatcher.video)
  219. except: pass
  220. try: os.remove("ffmpeg-output.yuv")
  221. except: pass
  222. return YUVOutputComparison
  223. class Regression(Case):
  224. depends = [ Compile ]
  225. _psnr_pattern = re.compile(r"x264 [[]info[]]: PSNR Mean Y:\d+[.]\d+ U:\d+[.]\d+ V:\d+[.]\d+ Avg:\d+[.]\d+ Global:(\d+[.]\d+) kb/s:\d+[.]\d+")
  226. _ssim_pattern = re.compile(r"x264 [[]info[]]: SSIM Mean Y:(\d+[.]\d+) [(]\d+[.]\d+db[)]")
  227. def __init__(self):
  228. if self.fixture.dispatcher.x264:
  229. self.__class__.__name__ += " %s" % " ".join(self.fixture.dispatcher.x264)
  230. def test_psnr(self):
  231. try:
  232. x264_proc = Popen([
  233. "./x264",
  234. "-o",
  235. "%s.264" % self.fixture.dispatcher.video,
  236. "--psnr"
  237. ] + self.fixture.dispatcher.x264 + [
  238. self.fixture.dispatcher.video
  239. ], stdout=PIPE, stderr=STDOUT)
  240. output = x264_proc.communicate()[0]
  241. if x264_proc.returncode != 0:
  242. raise FailedTestError("x264 did not complete properly: %s" % output.replace("\n", " "))
  243. for line in output.split("\n"):
  244. if line.startswith("x264 [info]: PSNR Mean"):
  245. return float(self._psnr_pattern.match(line).group(1))
  246. raise FailedTestError("no PSNR output caught from x264")
  247. finally:
  248. try: os.remove("%s.264" % self.fixture.dispatcher.video)
  249. except: pass
  250. def test_ssim(self):
  251. try:
  252. x264_proc = Popen([
  253. "./x264",
  254. "-o",
  255. "%s.264" % self.fixture.dispatcher.video,
  256. "--ssim"
  257. ] + self.fixture.dispatcher.x264 + [
  258. self.fixture.dispatcher.video
  259. ], stdout=PIPE, stderr=STDOUT)
  260. output = x264_proc.communicate()[0]
  261. if x264_proc.returncode != 0:
  262. raise FailedTestError("x264 did not complete properly: %s" % output.replace("\n", " "))
  263. for line in output.split("\n"):
  264. if line.startswith("x264 [info]: SSIM Mean"):
  265. return float(self._ssim_pattern.match(line).group(1))
  266. raise FailedTestError("no PSNR output caught from x264")
  267. finally:
  268. try: os.remove("%s.264" % self.fixture.dispatcher.video)
  269. except: pass
  270. def _generate_random_commandline():
  271. commandline = []
  272. for suboptions in OPTIONS:
  273. commandline.append(suboptions[randrange(0, len(suboptions))])
  274. return filter(None, reduce(operator.add, [ shlex.split(opt) for opt in commandline ]))
  275. _generated = []
  276. fixture = x264()
  277. fixture.register_case(Compile)
  278. fixture.register_case(Regression)
  279. class Dispatcher(_Dispatcher):
  280. video = "akiyo_qcif.y4m"
  281. products = 50
  282. configure = []
  283. x264 = []
  284. yuv_tests = [ "jm" ]
  285. def _populate_parser(self):
  286. super(Dispatcher, self)._populate_parser()
  287. # don't do a whole lot with this
  288. tcase = _YUVOutputComparisonFactory()
  289. yuv_tests = [ name[5:] for name, meth in filter(lambda pair: pair[0][:5] == "test_", inspect.getmembers(tcase)) ]
  290. group = OptionGroup(self.optparse, "x264 testing-specific options")
  291. group.add_option(
  292. "-v",
  293. "--video",
  294. metavar="FILENAME",
  295. action="callback",
  296. dest="video",
  297. type=str,
  298. callback=lambda option, opt, value, parser: setattr(self, "video", value),
  299. help="yuv video to perform testing on (default: %s)" % self.video
  300. )
  301. group.add_option(
  302. "-s",
  303. "--seed",
  304. metavar="SEED",
  305. action="callback",
  306. dest="seed",
  307. type=int,
  308. callback=lambda option, opt, value, parser: setattr(self, "seed", value),
  309. help="seed for the random number generator (default: unix timestamp)"
  310. )
  311. group.add_option(
  312. "-p",
  313. "--product-tests",
  314. metavar="NUM",
  315. action="callback",
  316. dest="video",
  317. type=int,
  318. callback=lambda option, opt, value, parser: setattr(self, "products", value),
  319. help="number of cartesian products to generate for yuv comparison testing (default: %d)" % self.products
  320. )
  321. group.add_option(
  322. "--configure-with",
  323. metavar="FLAGS",
  324. action="callback",
  325. dest="configure",
  326. type=str,
  327. callback=lambda option, opt, value, parser: setattr(self, "configure", shlex.split(value)),
  328. help="options to run ./configure with"
  329. )
  330. group.add_option(
  331. "--yuv-tests",
  332. action="callback",
  333. dest="yuv_tests",
  334. type=str,
  335. callback=lambda option, opt, value, parser: setattr(self, "yuv_tests", [
  336. val.strip() for val in value.split(",")
  337. ]),
  338. help="select tests to run with yuv comparisons (default: %s, available: %s)" % (
  339. ", ".join(self.yuv_tests),
  340. ", ".join(yuv_tests)
  341. )
  342. )
  343. group.add_option(
  344. "--x264-with",
  345. metavar="FLAGS",
  346. action="callback",
  347. dest="x264",
  348. type=str,
  349. callback=lambda option, opt, value, parser: setattr(self, "x264", shlex.split(value)),
  350. help="additional options to run ./x264 with"
  351. )
  352. self.optparse.add_option_group(group)
  353. def pre_dispatch(self):
  354. if not hasattr(self, "seed"):
  355. self.seed = int(time())
  356. print "Using seed: %d" % self.seed
  357. seed(self.seed)
  358. for i in xrange(self.products):
  359. YUVOutputComparison = _YUVOutputComparisonFactory()
  360. commandline = _generate_random_commandline()
  361. counter = 0
  362. while commandline in _generated:
  363. counter += 1
  364. commandline = _generate_random_commandline()
  365. if counter > 100:
  366. print >>sys.stderr, "Maximum command-line regeneration exceeded. " \
  367. "Try a different seed or specify fewer products to generate."
  368. sys.exit(1)
  369. commandline += self.x264
  370. _generated.append(commandline)
  371. YUVOutputComparison.options = commandline
  372. YUVOutputComparison.__name__ = ("%s %s" % (YUVOutputComparison.__name__, " ".join(commandline)))
  373. fixture.register_case(YUVOutputComparison)
  374. Dispatcher(fixture).dispatch()