set.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379
  1. /*****************************************************************************
  2. * set.c: quantization init
  3. *****************************************************************************
  4. * Copyright (C) 2005-2018 x264 project
  5. *
  6. * Authors: Loren Merritt <lorenm@u.washington.edu>
  7. *
  8. * This program is free software; you can redistribute it and/or modify
  9. * it under the terms of the GNU General Public License as published by
  10. * the Free Software Foundation; either version 2 of the License, or
  11. * (at your option) any later version.
  12. *
  13. * This program is distributed in the hope that it will be useful,
  14. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16. * GNU General Public License for more details.
  17. *
  18. * You should have received a copy of the GNU General Public License
  19. * along with this program; if not, write to the Free Software
  20. * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA.
  21. *
  22. * This program is also available under a commercial proprietary license.
  23. * For more information, contact us at licensing@x264.com.
  24. *****************************************************************************/
  25. #include "common.h"
  26. #define SHIFT(x,s) ((s)<=0 ? (x)<<-(s) : ((x)+(1<<((s)-1)))>>(s))
  27. #define DIV(n,d) (((n) + ((d)>>1)) / (d))
  28. static const uint8_t dequant4_scale[6][3] =
  29. {
  30. { 10, 13, 16 },
  31. { 11, 14, 18 },
  32. { 13, 16, 20 },
  33. { 14, 18, 23 },
  34. { 16, 20, 25 },
  35. { 18, 23, 29 }
  36. };
  37. static const uint16_t quant4_scale[6][3] =
  38. {
  39. { 13107, 8066, 5243 },
  40. { 11916, 7490, 4660 },
  41. { 10082, 6554, 4194 },
  42. { 9362, 5825, 3647 },
  43. { 8192, 5243, 3355 },
  44. { 7282, 4559, 2893 },
  45. };
  46. static const uint8_t quant8_scan[16] =
  47. {
  48. 0,3,4,3, 3,1,5,1, 4,5,2,5, 3,1,5,1
  49. };
  50. static const uint8_t dequant8_scale[6][6] =
  51. {
  52. { 20, 18, 32, 19, 25, 24 },
  53. { 22, 19, 35, 21, 28, 26 },
  54. { 26, 23, 42, 24, 33, 31 },
  55. { 28, 25, 45, 26, 35, 33 },
  56. { 32, 28, 51, 30, 40, 38 },
  57. { 36, 32, 58, 34, 46, 43 },
  58. };
  59. static const uint16_t quant8_scale[6][6] =
  60. {
  61. { 13107, 11428, 20972, 12222, 16777, 15481 },
  62. { 11916, 10826, 19174, 11058, 14980, 14290 },
  63. { 10082, 8943, 15978, 9675, 12710, 11985 },
  64. { 9362, 8228, 14913, 8931, 11984, 11259 },
  65. { 8192, 7346, 13159, 7740, 10486, 9777 },
  66. { 7282, 6428, 11570, 6830, 9118, 8640 }
  67. };
  68. int x264_cqm_init( x264_t *h )
  69. {
  70. int def_quant4[6][16];
  71. int def_quant8[6][64];
  72. int def_dequant4[6][16];
  73. int def_dequant8[6][64];
  74. int quant4_mf[4][6][16];
  75. int quant8_mf[4][6][64];
  76. int deadzone[4] = { 32 - h->param.analyse.i_luma_deadzone[1],
  77. 32 - h->param.analyse.i_luma_deadzone[0],
  78. 32 - 11, 32 - 21 };
  79. int max_qp_err = -1;
  80. int max_chroma_qp_err = -1;
  81. int min_qp_err = QP_MAX+1;
  82. int num_8x8_lists = h->sps->i_chroma_format_idc == CHROMA_444 ? 4
  83. : h->param.analyse.b_transform_8x8 ? 2 : 0; /* Checkasm may segfault if optimized out by --chroma-format */
  84. #define CQM_ALLOC( w, count )\
  85. for( int i = 0; i < count; i++ )\
  86. {\
  87. int size = w*w;\
  88. int start = w == 8 ? 4 : 0;\
  89. int j;\
  90. for( j = 0; j < i; j++ )\
  91. if( !memcmp( h->sps->scaling_list[i+start], h->sps->scaling_list[j+start], size*sizeof(uint8_t) ) )\
  92. break;\
  93. if( j < i )\
  94. {\
  95. h-> quant##w##_mf[i] = h-> quant##w##_mf[j];\
  96. h->dequant##w##_mf[i] = h->dequant##w##_mf[j];\
  97. h->unquant##w##_mf[i] = h->unquant##w##_mf[j];\
  98. }\
  99. else\
  100. {\
  101. CHECKED_MALLOC( h-> quant##w##_mf[i], (QP_MAX_SPEC+1)*size*sizeof(udctcoef) );\
  102. CHECKED_MALLOC( h->dequant##w##_mf[i], 6*size*sizeof(int) );\
  103. CHECKED_MALLOC( h->unquant##w##_mf[i], (QP_MAX_SPEC+1)*size*sizeof(int) );\
  104. }\
  105. for( j = 0; j < i; j++ )\
  106. if( deadzone[j] == deadzone[i] &&\
  107. !memcmp( h->sps->scaling_list[i+start], h->sps->scaling_list[j+start], size*sizeof(uint8_t) ) )\
  108. break;\
  109. if( j < i )\
  110. {\
  111. h->quant##w##_bias[i] = h->quant##w##_bias[j];\
  112. h->quant##w##_bias0[i] = h->quant##w##_bias0[j];\
  113. }\
  114. else\
  115. {\
  116. CHECKED_MALLOC( h->quant##w##_bias[i], (QP_MAX_SPEC+1)*size*sizeof(udctcoef) );\
  117. CHECKED_MALLOC( h->quant##w##_bias0[i], (QP_MAX_SPEC+1)*size*sizeof(udctcoef) );\
  118. }\
  119. }
  120. CQM_ALLOC( 4, 4 )
  121. CQM_ALLOC( 8, num_8x8_lists )
  122. for( int q = 0; q < 6; q++ )
  123. {
  124. for( int i = 0; i < 16; i++ )
  125. {
  126. int j = (i&1) + ((i>>2)&1);
  127. def_dequant4[q][i] = dequant4_scale[q][j];
  128. def_quant4[q][i] = quant4_scale[q][j];
  129. }
  130. for( int i = 0; i < 64; i++ )
  131. {
  132. int j = quant8_scan[((i>>1)&12) | (i&3)];
  133. def_dequant8[q][i] = dequant8_scale[q][j];
  134. def_quant8[q][i] = quant8_scale[q][j];
  135. }
  136. }
  137. for( int q = 0; q < 6; q++ )
  138. {
  139. for( int i_list = 0; i_list < 4; i_list++ )
  140. for( int i = 0; i < 16; i++ )
  141. {
  142. h->dequant4_mf[i_list][q][i] = def_dequant4[q][i] * h->sps->scaling_list[i_list][i];
  143. quant4_mf[i_list][q][i] = DIV(def_quant4[q][i] * 16, h->sps->scaling_list[i_list][i]);
  144. }
  145. for( int i_list = 0; i_list < num_8x8_lists; i_list++ )
  146. for( int i = 0; i < 64; i++ )
  147. {
  148. h->dequant8_mf[i_list][q][i] = def_dequant8[q][i] * h->sps->scaling_list[4+i_list][i];
  149. quant8_mf[i_list][q][i] = DIV(def_quant8[q][i] * 16, h->sps->scaling_list[4+i_list][i]);
  150. }
  151. }
  152. for( int q = 0; q <= QP_MAX_SPEC; q++ )
  153. {
  154. int j;
  155. for( int i_list = 0; i_list < 4; i_list++ )
  156. for( int i = 0; i < 16; i++ )
  157. {
  158. h->unquant4_mf[i_list][q][i] = (1ULL << (q/6 + 15 + 8)) / quant4_mf[i_list][q%6][i];
  159. h->quant4_mf[i_list][q][i] = j = SHIFT(quant4_mf[i_list][q%6][i], q/6 - 1);
  160. if( !j )
  161. {
  162. min_qp_err = X264_MIN( min_qp_err, q );
  163. continue;
  164. }
  165. // round to nearest, unless that would cause the deadzone to be negative
  166. h->quant4_bias[i_list][q][i] = X264_MIN( DIV(deadzone[i_list]<<10, j), (1<<15)/j );
  167. h->quant4_bias0[i_list][q][i] = (1<<15)/j;
  168. if( j > 0xffff && q > max_qp_err && (i_list == CQM_4IY || i_list == CQM_4PY) )
  169. max_qp_err = q;
  170. if( j > 0xffff && q > max_chroma_qp_err && (i_list == CQM_4IC || i_list == CQM_4PC) )
  171. max_chroma_qp_err = q;
  172. }
  173. if( h->param.analyse.b_transform_8x8 )
  174. for( int i_list = 0; i_list < num_8x8_lists; i_list++ )
  175. for( int i = 0; i < 64; i++ )
  176. {
  177. h->unquant8_mf[i_list][q][i] = (1ULL << (q/6 + 16 + 8)) / quant8_mf[i_list][q%6][i];
  178. j = SHIFT(quant8_mf[i_list][q%6][i], q/6);
  179. h->quant8_mf[i_list][q][i] = (uint16_t)j;
  180. if( !j )
  181. {
  182. min_qp_err = X264_MIN( min_qp_err, q );
  183. continue;
  184. }
  185. h->quant8_bias[i_list][q][i] = X264_MIN( DIV(deadzone[i_list]<<10, j), (1<<15)/j );
  186. h->quant8_bias0[i_list][q][i] = (1<<15)/j;
  187. if( j > 0xffff && q > max_qp_err && (i_list == CQM_8IY || i_list == CQM_8PY) )
  188. max_qp_err = q;
  189. if( j > 0xffff && q > max_chroma_qp_err && (i_list == CQM_8IC || i_list == CQM_8PC) )
  190. max_chroma_qp_err = q;
  191. }
  192. }
  193. /* Emergency mode denoising. */
  194. x264_emms();
  195. CHECKED_MALLOC( h->nr_offset_emergency, sizeof(*h->nr_offset_emergency)*(QP_MAX-QP_MAX_SPEC) );
  196. for( int q = 0; q < QP_MAX - QP_MAX_SPEC; q++ )
  197. for( int cat = 0; cat < 3 + CHROMA444; cat++ )
  198. {
  199. int dct8x8 = cat&1;
  200. if( !h->param.analyse.b_transform_8x8 && dct8x8 )
  201. continue;
  202. int size = dct8x8 ? 64 : 16;
  203. udctcoef *nr_offset = h->nr_offset_emergency[q][cat];
  204. /* Denoise chroma first (due to h264's chroma QP offset), then luma, then DC. */
  205. int dc_threshold = (QP_MAX-QP_MAX_SPEC)*2/3;
  206. int luma_threshold = (QP_MAX-QP_MAX_SPEC)*2/3;
  207. int chroma_threshold = 0;
  208. for( int i = 0; i < size; i++ )
  209. {
  210. int max = (1 << (7 + BIT_DEPTH)) - 1;
  211. /* True "emergency mode": remove all DCT coefficients */
  212. if( q == QP_MAX - QP_MAX_SPEC - 1 )
  213. {
  214. nr_offset[i] = max;
  215. continue;
  216. }
  217. int thresh = i == 0 ? dc_threshold : cat >= 2 ? chroma_threshold : luma_threshold;
  218. if( q < thresh )
  219. {
  220. nr_offset[i] = 0;
  221. continue;
  222. }
  223. double pos = (double)(q-thresh+1) / (QP_MAX - QP_MAX_SPEC - thresh);
  224. /* XXX: this math is largely tuned for /dev/random input. */
  225. double start = dct8x8 ? h->unquant8_mf[CQM_8PY][QP_MAX_SPEC][i]
  226. : h->unquant4_mf[CQM_4PY][QP_MAX_SPEC][i];
  227. /* Formula chosen as an exponential scale to vaguely mimic the effects
  228. * of a higher quantizer. */
  229. double bias = (pow( 2, pos*(QP_MAX - QP_MAX_SPEC)/10. )*0.003-0.003) * start;
  230. nr_offset[i] = X264_MIN( bias + 0.5, max );
  231. }
  232. }
  233. if( !h->mb.b_lossless )
  234. {
  235. while( h->chroma_qp_table[SPEC_QP(h->param.rc.i_qp_min)] <= max_chroma_qp_err )
  236. h->param.rc.i_qp_min++;
  237. if( min_qp_err <= h->param.rc.i_qp_max )
  238. h->param.rc.i_qp_max = min_qp_err-1;
  239. if( max_qp_err >= h->param.rc.i_qp_min )
  240. h->param.rc.i_qp_min = max_qp_err+1;
  241. /* If long level-codes aren't allowed, we need to allow QP high enough to avoid them. */
  242. if( !h->param.b_cabac && h->sps->i_profile_idc < PROFILE_HIGH )
  243. while( h->chroma_qp_table[SPEC_QP(h->param.rc.i_qp_max)] <= 12 || h->param.rc.i_qp_max <= 12 )
  244. h->param.rc.i_qp_max++;
  245. if( h->param.rc.i_qp_min > h->param.rc.i_qp_max )
  246. {
  247. x264_log( h, X264_LOG_ERROR, "Impossible QP constraints for CQM (min=%d, max=%d)\n", h->param.rc.i_qp_min, h->param.rc.i_qp_max );
  248. return -1;
  249. }
  250. }
  251. return 0;
  252. fail:
  253. x264_cqm_delete( h );
  254. return -1;
  255. }
  256. #define CQM_DELETE( n, max )\
  257. for( int i = 0; i < (max); i++ )\
  258. {\
  259. int j;\
  260. for( j = 0; j < i; j++ )\
  261. if( h->quant##n##_mf[i] == h->quant##n##_mf[j] )\
  262. break;\
  263. if( j == i )\
  264. {\
  265. x264_free( h-> quant##n##_mf[i] );\
  266. x264_free( h->dequant##n##_mf[i] );\
  267. x264_free( h->unquant##n##_mf[i] );\
  268. }\
  269. for( j = 0; j < i; j++ )\
  270. if( h->quant##n##_bias[i] == h->quant##n##_bias[j] )\
  271. break;\
  272. if( j == i )\
  273. {\
  274. x264_free( h->quant##n##_bias[i] );\
  275. x264_free( h->quant##n##_bias0[i] );\
  276. }\
  277. }
  278. void x264_cqm_delete( x264_t *h )
  279. {
  280. CQM_DELETE( 4, 4 );
  281. CQM_DELETE( 8, CHROMA444 ? 4 : 2 );
  282. x264_free( h->nr_offset_emergency );
  283. }
  284. static int cqm_parse_jmlist( x264_t *h, const char *buf, const char *name,
  285. uint8_t *cqm, const uint8_t *jvt, int length )
  286. {
  287. int i;
  288. char *p = strstr( buf, name );
  289. if( !p )
  290. {
  291. memset( cqm, 16, length );
  292. return 0;
  293. }
  294. p += strlen( name );
  295. if( *p == 'U' || *p == 'V' )
  296. p++;
  297. char *nextvar = strstr( p, "INT" );
  298. for( i = 0; i < length && (p = strpbrk( p, " \t\n," )) && (p = strpbrk( p, "0123456789" )); i++ )
  299. {
  300. int coef = -1;
  301. sscanf( p, "%d", &coef );
  302. if( i == 0 && coef == 0 )
  303. {
  304. memcpy( cqm, jvt, length );
  305. return 0;
  306. }
  307. if( coef < 1 || coef > 255 )
  308. {
  309. x264_log( h, X264_LOG_ERROR, "bad coefficient in list '%s'\n", name );
  310. return -1;
  311. }
  312. cqm[i] = coef;
  313. }
  314. if( (nextvar && p > nextvar) || i != length )
  315. {
  316. x264_log( h, X264_LOG_ERROR, "not enough coefficients in list '%s'\n", name );
  317. return -1;
  318. }
  319. return 0;
  320. }
  321. int x264_cqm_parse_file( x264_t *h, const char *filename )
  322. {
  323. char *p;
  324. int b_error = 0;
  325. h->param.i_cqm_preset = X264_CQM_CUSTOM;
  326. char *buf = x264_slurp_file( filename );
  327. if( !buf )
  328. {
  329. x264_log( h, X264_LOG_ERROR, "can't open file '%s'\n", filename );
  330. return -1;
  331. }
  332. while( (p = strchr( buf, '#' )) != NULL )
  333. memset( p, ' ', strcspn( p, "\n" ) );
  334. b_error |= cqm_parse_jmlist( h, buf, "INTRA4X4_LUMA", h->param.cqm_4iy, x264_cqm_jvt4i, 16 );
  335. b_error |= cqm_parse_jmlist( h, buf, "INTER4X4_LUMA", h->param.cqm_4py, x264_cqm_jvt4p, 16 );
  336. b_error |= cqm_parse_jmlist( h, buf, "INTRA4X4_CHROMA", h->param.cqm_4ic, x264_cqm_jvt4i, 16 );
  337. b_error |= cqm_parse_jmlist( h, buf, "INTER4X4_CHROMA", h->param.cqm_4pc, x264_cqm_jvt4p, 16 );
  338. b_error |= cqm_parse_jmlist( h, buf, "INTRA8X8_LUMA", h->param.cqm_8iy, x264_cqm_jvt8i, 64 );
  339. b_error |= cqm_parse_jmlist( h, buf, "INTER8X8_LUMA", h->param.cqm_8py, x264_cqm_jvt8p, 64 );
  340. if( CHROMA444 )
  341. {
  342. b_error |= cqm_parse_jmlist( h, buf, "INTRA8X8_CHROMA", h->param.cqm_8ic, x264_cqm_jvt8i, 64 );
  343. b_error |= cqm_parse_jmlist( h, buf, "INTER8X8_CHROMA", h->param.cqm_8pc, x264_cqm_jvt8p, 64 );
  344. }
  345. x264_free( buf );
  346. return b_error;
  347. }