diff --git a/include/freerdp/codec/yuv.h b/include/freerdp/codec/yuv.h index ef36f03d4..ab66cd7f7 100644 --- a/include/freerdp/codec/yuv.h +++ b/include/freerdp/codec/yuv.h @@ -33,19 +33,19 @@ extern "C" #endif FREERDP_API BOOL yuv420_context_decode(YUV_CONTEXT* context, const BYTE* pYUVData[3], - const UINT32 iStride[3], DWORD DstFormat, BYTE* dest, - UINT32 nDstStep, const RECTANGLE_16* regionRects, - UINT32 numRegionRects); + const UINT32 iStride[3], UINT32 yuvHeight, + DWORD DstFormat, BYTE* dest, UINT32 nDstStep, + const RECTANGLE_16* regionRects, UINT32 numRegionRects); FREERDP_API BOOL yuv420_context_encode(YUV_CONTEXT* context, const BYTE* rgbData, UINT32 srcStep, UINT32 srcFormat, const UINT32 iStride[3], BYTE* yuvData[3], const RECTANGLE_16* regionRects, UINT32 numRegionRects); FREERDP_API BOOL yuv444_context_decode(YUV_CONTEXT* context, BYTE type, const BYTE* pYUVData[3], - const UINT32 iStride[3], BYTE* pYUVDstData[3], - const UINT32 iDstStride[3], DWORD DstFormat, BYTE* dest, - UINT32 nDstStep, const RECTANGLE_16* regionRects, - UINT32 numRegionRects); + const UINT32 iStride[3], UINT32 srcYuvHeight, + BYTE* pYUVDstData[3], const UINT32 iDstStride[3], + DWORD DstFormat, BYTE* dest, UINT32 nDstStep, + const RECTANGLE_16* regionRects, UINT32 numRegionRects); FREERDP_API BOOL yuv444_context_encode(YUV_CONTEXT* context, BYTE version, const BYTE* pSrcData, UINT32 nSrcStep, UINT32 SrcFormat, const UINT32 iStride[3], BYTE* pYUVLumaData[3], diff --git a/libfreerdp/codec/h264.c b/libfreerdp/codec/h264.c index ad7028a23..531ae5784 100644 --- a/libfreerdp/codec/h264.c +++ b/libfreerdp/codec/h264.c @@ -109,8 +109,8 @@ INT32 avc420_decompress(H264_CONTEXT* h264, const BYTE* pSrcData, UINT32 SrcSize pYUVData[0] = h264->pYUVData[0]; pYUVData[1] = h264->pYUVData[1]; pYUVData[2] = h264->pYUVData[2]; - if (!yuv420_context_decode(h264->yuv, pYUVData, h264->iStride, DstFormat, pDstData, nDstStep, - regionRects, numRegionRects)) + if (!yuv420_context_decode(h264->yuv, pYUVData, h264->iStride, h264->height, DstFormat, + pDstData, nDstStep, regionRects, numRegionRects)) return -1002; return 1; @@ -450,8 +450,8 @@ static BOOL avc444_process_rects(H264_CONTEXT* h264, const BYTE* pSrcData, UINT3 pYUVDstData[0] = ppYUVDstData[0]; pYUVDstData[1] = ppYUVDstData[1]; pYUVDstData[2] = ppYUVDstData[2]; - if (!yuv444_context_decode(h264->yuv, type, pYUVData, piStride, pYUVDstData, piDstStride, - DstFormat, pDstData, nDstStep, rects, nrRects)) + if (!yuv444_context_decode(h264->yuv, type, pYUVData, piStride, h264->height, pYUVDstData, + piDstStride, DstFormat, pDstData, nDstStep, rects, nrRects)) return FALSE; return TRUE; diff --git a/libfreerdp/codec/yuv.c b/libfreerdp/codec/yuv.c index 53d67b59f..83fac924b 100644 --- a/libfreerdp/codec/yuv.c +++ b/libfreerdp/codec/yuv.c @@ -269,8 +269,8 @@ static void free_objects(PTP_WORK* work_objects, void* params, UINT32 waitCount) } static BOOL pool_decode(YUV_CONTEXT* context, PTP_WORK_CALLBACK cb, const BYTE* pYUVData[3], - const UINT32 iStride[3], UINT32 DstFormat, BYTE* dest, UINT32 nDstStep, - const RECTANGLE_16* regionRects, UINT32 numRegionRects) + const UINT32 iStride[3], UINT32 yuvHeight, UINT32 DstFormat, BYTE* dest, + UINT32 nDstStep, const RECTANGLE_16* regionRects, UINT32 numRegionRects) { UINT32 steps; BOOL rc = FALSE; @@ -318,6 +318,8 @@ static BOOL pool_decode(YUV_CONTEXT* context, PTP_WORK_CALLBACK cb, const BYTE* r.bottom = rect->bottom; if (r.top >= rect->bottom) continue; + if (r.bottom > yuvHeight) + r.bottom = yuvHeight; *cur = pool_decode_param(&r, context, pYUVData, iStride, DstFormat, dest, nDstStep); if (!submit_object(&work_objects[waitCount], cb, cur, context)) goto fail; @@ -375,7 +377,7 @@ static void CALLBACK yuv444_combine_work_callback(PTP_CALLBACK_INSTANCE instance static INLINE YUV_COMBINE_WORK_PARAM pool_decode_rect_param( const RECTANGLE_16* rect, YUV_CONTEXT* context, BYTE type, const BYTE* pYUVData[3], - const UINT32 iStride[3], BYTE* pYUVDstData[3], const UINT32 iDstStride[3]) + const UINT32 iStride[3], UINT32 yuvHeight, BYTE* pYUVDstData[3], const UINT32 iDstStride[3]) { YUV_COMBINE_WORK_PARAM current = { 0 }; current.context = context; @@ -397,7 +399,7 @@ static INLINE YUV_COMBINE_WORK_PARAM pool_decode_rect_param( } static BOOL pool_decode_rect(YUV_CONTEXT* context, BYTE type, const BYTE* pYUVData[3], - const UINT32 iStride[3], BYTE* pYUVDstData[3], + const UINT32 iStride[3], UINT32 yuvHeight, BYTE* pYUVDstData[3], const UINT32 iDstStride[3], const RECTANGLE_16* regionRects, UINT32 numRegionRects) { @@ -413,8 +415,9 @@ static BOOL pool_decode_rect(YUV_CONTEXT* context, BYTE type, const BYTE* pYUVDa { for (y = 0; y < numRegionRects; y++) { - YUV_COMBINE_WORK_PARAM current = pool_decode_rect_param( - ®ionRects[y], context, type, pYUVData, iStride, pYUVDstData, iDstStride); + YUV_COMBINE_WORK_PARAM current = + pool_decode_rect_param(®ionRects[y], context, type, pYUVData, iStride, yuvHeight, + pYUVDstData, iDstStride); cb(NULL, ¤t, NULL); } return TRUE; @@ -429,7 +432,7 @@ static BOOL pool_decode_rect(YUV_CONTEXT* context, BYTE type, const BYTE* pYUVDa { YUV_COMBINE_WORK_PARAM* current = ¶ms[waitCount]; *current = pool_decode_rect_param(®ionRects[waitCount], context, type, pYUVData, iStride, - pYUVDstData, iDstStride); + yuvHeight, pYUVDstData, iDstStride); if (!submit_object(&work_objects[waitCount], cb, current, context)) goto fail; @@ -442,29 +445,29 @@ fail: } BOOL yuv444_context_decode(YUV_CONTEXT* context, BYTE type, const BYTE* pYUVData[3], - const UINT32 iStride[3], BYTE* pYUVDstData[3], + const UINT32 iStride[3], UINT32 yuvHeight, BYTE* pYUVDstData[3], const UINT32 iDstStride[3], DWORD DstFormat, BYTE* dest, UINT32 nDstStep, const RECTANGLE_16* regionRects, UINT32 numRegionRects) { const BYTE* pYUVCDstData[3]; - if (!pool_decode_rect(context, type, pYUVData, iStride, pYUVDstData, iDstStride, regionRects, - numRegionRects)) + if (!pool_decode_rect(context, type, pYUVData, iStride, yuvHeight, pYUVDstData, iDstStride, + regionRects, numRegionRects)) return FALSE; pYUVCDstData[0] = pYUVDstData[0]; pYUVCDstData[1] = pYUVDstData[1]; pYUVCDstData[2] = pYUVDstData[2]; - return pool_decode(context, yuv444_process_work_callback, pYUVCDstData, iDstStride, DstFormat, - dest, nDstStep, regionRects, numRegionRects); + return pool_decode(context, yuv444_process_work_callback, pYUVCDstData, iDstStride, yuvHeight, + DstFormat, dest, nDstStep, regionRects, numRegionRects); } BOOL yuv420_context_decode(YUV_CONTEXT* context, const BYTE* pYUVData[3], const UINT32 iStride[3], - DWORD DstFormat, BYTE* dest, UINT32 nDstStep, + UINT32 yuvHeight, DWORD DstFormat, BYTE* dest, UINT32 nDstStep, const RECTANGLE_16* regionRects, UINT32 numRegionRects) { - return pool_decode(context, yuv420_process_work_callback, pYUVData, iStride, DstFormat, dest, - nDstStep, regionRects, numRegionRects); + return pool_decode(context, yuv420_process_work_callback, pYUVData, iStride, yuvHeight, + DstFormat, dest, nDstStep, regionRects, numRegionRects); } static void CALLBACK yuv420_encode_work_callback(PTP_CALLBACK_INSTANCE instance, void* context,