AOMedia AV1 Codec
temporal_filter.h
1 /*
2  * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3  *
4  * This source code is subject to the terms of the BSD 2 Clause License and
5  * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6  * was not distributed with this source code in the LICENSE file, you can
7  * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8  * Media Patent License 1.0 was not distributed with this source code in the
9  * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10  */
11 
12 #ifndef AOM_AV1_ENCODER_TEMPORAL_FILTER_H_
13 #define AOM_AV1_ENCODER_TEMPORAL_FILTER_H_
14 
15 #ifdef __cplusplus
16 extern "C" {
17 #endif
19 struct AV1_COMP;
20 struct AV1EncoderConfig;
21 struct ThreadData;
22 // TODO(any): These two variables are only used in avx2, sse2, sse4
23 // implementations, where the block size is still hard coded. This should be
24 // fixed to align with the c implementation.
25 #define BH 32
26 #define BW 32
27 
28 // Block size used in temporal filtering.
29 #define TF_BLOCK_SIZE BLOCK_32X32
30 
31 // Window size for temporal filtering.
32 #define TF_WINDOW_LENGTH 5
33 
34 // Hyper-parameters used to compute filtering weight. These hyper-parameters can
35 // be tuned for a better performance.
36 // 0. A scale factor used in temporal filtering to raise the filter weight from
37 // `double` with range [0, 1] to `int` with range [0, 1000].
38 #define TF_WEIGHT_SCALE 1000
39 // 1. Weight factor used to balance the weighted-average between window error
40 // and block error. The weight is for window error while the weight for block
41 // error is always set as 1.
42 #define TF_WINDOW_BLOCK_BALANCE_WEIGHT 5
43 // 2. Threshold for using q to adjust the filtering weight. Concretely, when
44 // using a small q (high bitrate), we would like to reduce the filtering
45 // strength such that more detailed information can be preserved. Hence, when
46 // q is smaller than this threshold, we will adjust the filtering weight
47 // based on the q-value.
48 #define TF_Q_DECAY_THRESHOLD 20
49 // 3. Normalization factor used to normalize the motion search error. Since the
50 // motion search error can be large and uncontrollable, we will simply
51 // normalize it before using it to compute the filtering weight.
52 #define TF_SEARCH_ERROR_NORM_WEIGHT 20
53 // 4. Threshold for using `arnr_strength` to adjust the filtering strength.
54 // Concretely, users can use `arnr_strength` arguments to control the
55 // strength of temporal filtering. When `arnr_strength` is small enough (
56 // i.e., smaller than this threshold), we will adjust the filtering weight
57 // based on the strength value.
58 #define TF_STRENGTH_THRESHOLD 4
59 // 5. Threshold for using motion search distance to adjust the filtering weight.
60 // Concretely, larger motion search vector leads to a higher probability of
61 // unreliable search. Hence, we would like to reduce the filtering strength
62 // when the distance is large enough. Considering that the distance actually
63 // relies on the frame size, this threshold is also a resolution-based
64 // threshold. Taking 720p videos as an instance, if this field equals to 0.1,
65 // then the actual threshold will be 720 * 0.1 = 72. Similarly, the threshold
66 // for 360p videos will be 360 * 0.1 = 36.
67 #define TF_SEARCH_DISTANCE_THRESHOLD 0.1
68 // 6. Threshold to identify if the q is in a relative high range.
69 // Above this cutoff q, a stronger filtering is applied.
70 // For a high q, the quantization throws away more information, and thus a
71 // stronger filtering is less likely to distort the encoded quality, while a
72 // stronger filtering could reduce bit rates.
73 // Ror a low q, more details are expected to be retained. Filtering is thus
74 // more conservative.
75 #define TF_QINDEX_CUTOFF 128
76 
77 #define NOISE_ESTIMATION_EDGE_THRESHOLD 50
78 
79 // Sum and SSE source vs filtered frame difference returned by
80 // temporal filter.
81 typedef struct {
82  int64_t sum;
83  int64_t sse;
84 } FRAME_DIFF;
85 
91 typedef struct {
95  YV12_BUFFER_CONFIG *frames[MAX_LAG_BUFFERS];
100 
105 
117  struct scale_factors sf;
121  double noise_levels[MAX_MB_PLANE];
125  int num_pels;
129  int mb_rows;
133  int mb_cols;
141  int q_factor;
143 
149 #define TF_INFO_BUF_COUNT 2
150 
154 typedef struct TEMPORAL_FILTER_INFO {
165  YV12_BUFFER_CONFIG tf_buf[TF_INFO_BUF_COUNT];
166 
177  FRAME_DIFF frame_diff[TF_INFO_BUF_COUNT];
181  int tf_buf_gf_index[TF_INFO_BUF_COUNT];
185  int tf_buf_display_index_offset[TF_INFO_BUF_COUNT];
189  int tf_buf_valid[TF_INFO_BUF_COUNT];
191 
197 int av1_is_temporal_filter_on(const struct AV1EncoderConfig *oxcf);
198 
203 void av1_tf_info_alloc(TEMPORAL_FILTER_INFO *tf_info, struct AV1_COMP *cpi);
204 
208 void av1_tf_info_free(TEMPORAL_FILTER_INFO *tf_info);
209 
213 void av1_tf_info_reset(TEMPORAL_FILTER_INFO *tf_info);
214 
220 void av1_tf_info_filtering(TEMPORAL_FILTER_INFO *tf_info, struct AV1_COMP *cpi,
221  const GF_GROUP *gf_group);
222 
229 YV12_BUFFER_CONFIG *av1_tf_info_get_filtered_buf(TEMPORAL_FILTER_INFO *tf_info,
230  int gf_index,
231  FRAME_DIFF *frame_diff);
232 
235 // Data related to temporal filtering.
236 typedef struct {
237  // Source vs filtered frame error.
238  FRAME_DIFF diff;
239  // Pointer to temporary block info used to store state in temporal filtering
240  // process.
241  MB_MODE_INFO *tmp_mbmi;
242  // Pointer to accumulator buffer used in temporal filtering process.
243  uint32_t *accum;
244  // Pointer to count buffer used in temporal filtering process.
245  uint16_t *count;
246  // Pointer to predictor used in temporal filtering process.
247  uint8_t *pred;
248 } TemporalFilterData;
249 
250 // Data related to temporal filter multi-thread synchronization.
251 typedef struct {
252 #if CONFIG_MULTITHREAD
253  // Mutex lock used for dispatching jobs.
254  pthread_mutex_t *mutex_;
255 #endif // CONFIG_MULTITHREAD
256  // Next temporal filter block row to be filtered.
257  int next_tf_row;
258 } AV1TemporalFilterSync;
259 
260 // Estimates noise level from a given frame using a single plane (Y, U, or V).
261 // This is an adaptation of the mehtod in the following paper:
262 // Shen-Chuan Tai, Shih-Ming Yang, "A fast method for image noise
263 // estimation using Laplacian operator and adaptive edge detection",
264 // Proc. 3rd International Symposium on Communications, Control and
265 // Signal Processing, 2008, St Julians, Malta.
266 // Inputs:
267 // frame: Pointer to the frame to estimate noise level from.
268 // plane: Index of the plane used for noise estimation. Commonly, 0 for
269 // Y-plane, 1 for U-plane, and 2 for V-plane.
270 // bit_depth: Actual bit-depth instead of the encoding bit-depth of the frame.
271 // Returns:
272 // The estimated noise, or -1.0 if there are too few smooth pixels.
273 double av1_estimate_noise_from_single_plane(const YV12_BUFFER_CONFIG *frame,
274  const int plane,
275  const int bit_depth,
276  const int edge_thresh);
290 void av1_tf_do_filtering_row(struct AV1_COMP *cpi, struct ThreadData *td,
291  int mb_row);
292 
317 void av1_temporal_filter(struct AV1_COMP *cpi,
318  const int filter_frame_lookahead_idx,
319  int gf_frame_index, FRAME_DIFF *frame_diff,
320  YV12_BUFFER_CONFIG *output_frame);
321 
337  const FRAME_DIFF *frame_diff, int q_index,
338  aom_bit_depth_t bit_depth);
339 
341 // Helper function to get `q` used for encoding.
342 int av1_get_q(const struct AV1_COMP *cpi);
343 
344 // Allocates memory for members of TemporalFilterData.
345 // Inputs:
346 // tf_data: Pointer to the structure containing temporal filter related data.
347 // num_pels: Number of pixels in the block across all planes.
348 // is_high_bitdepth: Whether the frame is high-bitdepth or not.
349 // Returns:
350 // Nothing will be returned. But the contents of tf_data will be modified.
351 static AOM_INLINE void tf_alloc_and_reset_data(TemporalFilterData *tf_data,
352  int num_pels,
353  int is_high_bitdepth) {
354  tf_data->tmp_mbmi = (MB_MODE_INFO *)malloc(sizeof(*tf_data->tmp_mbmi));
355  memset(tf_data->tmp_mbmi, 0, sizeof(*tf_data->tmp_mbmi));
356  tf_data->accum =
357  (uint32_t *)aom_memalign(16, num_pels * sizeof(*tf_data->accum));
358  tf_data->count =
359  (uint16_t *)aom_memalign(16, num_pels * sizeof(*tf_data->count));
360  memset(&tf_data->diff, 0, sizeof(tf_data->diff));
361  if (is_high_bitdepth)
362  tf_data->pred = CONVERT_TO_BYTEPTR(
363  aom_memalign(32, num_pels * 2 * sizeof(*tf_data->pred)));
364  else
365  tf_data->pred =
366  (uint8_t *)aom_memalign(32, num_pels * sizeof(*tf_data->pred));
367 }
368 
369 // Setup macroblockd params for temporal filtering process.
370 // Inputs:
371 // mbd: Pointer to the block for filtering.
372 // tf_data: Pointer to the structure containing temporal filter related data.
373 // scale: Scaling factor.
374 // Returns:
375 // Nothing will be returned. Contents of mbd will be modified.
376 static AOM_INLINE void tf_setup_macroblockd(MACROBLOCKD *mbd,
377  TemporalFilterData *tf_data,
378  const struct scale_factors *scale) {
379  mbd->block_ref_scale_factors[0] = scale;
380  mbd->block_ref_scale_factors[1] = scale;
381  mbd->mi = &tf_data->tmp_mbmi;
382  mbd->mi[0]->motion_mode = SIMPLE_TRANSLATION;
383 }
384 
385 // Deallocates the memory allocated for members of TemporalFilterData.
386 // Inputs:
387 // tf_data: Pointer to the structure containing temporal filter related data.
388 // is_high_bitdepth: Whether the frame is high-bitdepth or not.
389 // Returns:
390 // Nothing will be returned.
391 static AOM_INLINE void tf_dealloc_data(TemporalFilterData *tf_data,
392  int is_high_bitdepth) {
393  if (is_high_bitdepth)
394  tf_data->pred = (uint8_t *)CONVERT_TO_SHORTPTR(tf_data->pred);
395  free(tf_data->tmp_mbmi);
396  aom_free(tf_data->accum);
397  aom_free(tf_data->count);
398  aom_free(tf_data->pred);
399 }
400 
401 // Saves the state prior to temporal filter process.
402 // Inputs:
403 // mbd: Pointer to the block for filtering.
404 // input_mbmi: Backup block info to save input state.
405 // input_buffer: Backup buffer pointer to save input state.
406 // num_planes: Number of planes.
407 // Returns:
408 // Nothing will be returned. Contents of input_mbmi and input_buffer will be
409 // modified.
410 static INLINE void tf_save_state(MACROBLOCKD *mbd, MB_MODE_INFO ***input_mbmi,
411  uint8_t **input_buffer, int num_planes) {
412  for (int i = 0; i < num_planes; i++) {
413  input_buffer[i] = mbd->plane[i].pre[0].buf;
414  }
415  *input_mbmi = mbd->mi;
416 }
417 
418 // Restores the initial state after temporal filter process.
419 // Inputs:
420 // mbd: Pointer to the block for filtering.
421 // input_mbmi: Backup block info from where input state is restored.
422 // input_buffer: Backup buffer pointer from where input state is restored.
423 // num_planes: Number of planes.
424 // Returns:
425 // Nothing will be returned. Contents of mbd will be modified.
426 static INLINE void tf_restore_state(MACROBLOCKD *mbd, MB_MODE_INFO **input_mbmi,
427  uint8_t **input_buffer, int num_planes) {
428  for (int i = 0; i < num_planes; i++) {
429  mbd->plane[i].pre[0].buf = input_buffer[i];
430  }
431  mbd->mi = input_mbmi;
432 }
433 
435 #ifdef __cplusplus
436 } // extern "C"
437 #endif
438 
439 #endif // AOM_AV1_ENCODER_TEMPORAL_FILTER_H_
enum aom_bit_depth aom_bit_depth_t
Bit depth for codecThis enumeration determines the bit depth of the codec.
int av1_check_show_filtered_frame(const YV12_BUFFER_CONFIG *frame, const FRAME_DIFF *frame_diff, int q_index, aom_bit_depth_t bit_depth)
Check whether a filtered frame can be show directly.
void av1_temporal_filter(struct AV1_COMP *cpi, const int filter_frame_lookahead_idx, int gf_frame_index, FRAME_DIFF *frame_diff, YV12_BUFFER_CONFIG *output_frame)
Performs temporal filtering if needed on a source frame. For example to create a filtered alternate r...
void av1_tf_do_filtering_row(struct AV1_COMP *cpi, struct ThreadData *td, int mb_row)
Does temporal filter for a given macroblock row.
Definition: temporal_filter.c:764
Main encoder configuration data structure.
Definition: encoder.h:894
Top level encoder structure.
Definition: encoder.h:2632
Data related to the current GF/ARF group and the individual frames within the group.
Definition: firstpass.h:344
Stores the prediction/txfm mode of the current coding block.
Definition: blockd.h:222
MOTION_MODE motion_mode
The motion mode used by the inter prediction.
Definition: blockd.h:250
Temporal filter info for a gop.
Definition: temporal_filter.h:154
int tf_buf_display_index_offset[2]
Definition: temporal_filter.h:185
YV12_BUFFER_CONFIG tf_buf[2]
Definition: temporal_filter.h:165
int tf_buf_gf_index[2]
Definition: temporal_filter.h:181
FRAME_DIFF frame_diff[2]
Definition: temporal_filter.h:177
int is_temporal_filter_on
Definition: temporal_filter.h:160
int tf_buf_valid[2]
Definition: temporal_filter.h:189
YV12_BUFFER_CONFIG tf_buf_second_arf
Definition: temporal_filter.h:173
Parameters related to temporal filtering.
Definition: temporal_filter.h:91
YV12_BUFFER_CONFIG * output_frame
Definition: temporal_filter.h:104
int q_factor
Definition: temporal_filter.h:141
int num_pels
Definition: temporal_filter.h:125
int num_frames
Definition: temporal_filter.h:99
int compute_frame_diff
Definition: temporal_filter.h:113
int mb_rows
Definition: temporal_filter.h:129
int mb_cols
Definition: temporal_filter.h:133
int is_highbitdepth
Definition: temporal_filter.h:137
int filter_frame_idx
Definition: temporal_filter.h:109
Variables related to current coding block.
Definition: blockd.h:577
struct macroblockd_plane plane[3]
Definition: blockd.h:613
const struct scale_factors * block_ref_scale_factors[2]
Definition: blockd.h:694
MB_MODE_INFO ** mi
Definition: blockd.h:624
YV12 frame buffer data structure.
Definition: yv12config.h:39