From: Laurent Aimar <fenrir@videolan.org>
Date: Sat, 12 Jun 2010 23:25:43 +0000 (+0200)
Subject: Fixed potential unaligned access in vaapi/dxva2 picture copy.
X-Git-Url: http://git.videolan.org/?p=vlc%2Fvlc-1.1.git;a=commitdiff_plain;h=3e949628bdd9fd17a6b43d612f70902a48d92804

Fixed potential unaligned access in vaapi/dxva2 picture copy.

It (probably) happens only when the video is non mod 16, or non mod 32
with YV12 hardware surface.
(cherry picked from commit 957409b26fea130c93c740dbb1a032b1e7624794)

Close #3606

Signed-off-by: Jean-Baptiste Kempf <jb@videolan.org>
---

diff --git a/modules/codec/avcodec/copy.c b/modules/codec/avcodec/copy.c
index 6bc0d82..9c9a44b 100644
--- a/modules/codec/avcodec/copy.c
+++ b/modules/codec/avcodec/copy.c
@@ -63,7 +63,6 @@
  */
 static void CopyFromUswc(uint8_t *dst, size_t dst_pitch,
                          const uint8_t *src, size_t src_pitch,
-                         unsigned unaligned,
                          unsigned width, unsigned height,
                          unsigned cpu)
 {
@@ -71,6 +70,7 @@ static void CopyFromUswc(uint8_t *dst, size_t dst_pitch,
 
     ASM_SSE2(cpu, "mfence");
     for (unsigned y = 0; y < height; y++) {
+        const unsigned unaligned = (intptr_t)src & 0x0f;
         unsigned x;
 
         for (x = 0; x < unaligned; x++)
@@ -237,13 +237,11 @@ static void CopyPlane(uint8_t *dst, size_t dst_pitch, const uint8_t *src, size_t
     assert(hstep > 0);
 
     for (unsigned y = 0; y < height; y += hstep) {
-        const unsigned unaligned = (intptr_t)src & 0x0f;
         const unsigned hblock =  __MIN(hstep, height - y);
 
         /* Copy a bunch of line into our cache */
         CopyFromUswc(cache, w16,
                      src, src_pitch,
-                     unaligned,
                      width, hblock, cpu);
 
         /* Copy from our cache to the destination */
@@ -270,13 +268,11 @@ static void SplitPlanes(uint8_t *dstu, size_t dstu_pitch,
     assert(hstep > 0);
 
     for (unsigned y = 0; y < height; y += hstep) {
-        const unsigned unaligned = (intptr_t)src & 0x0f;
         const unsigned hblock =  __MIN(hstep, height - y);
 
         /* Copy a bunch of line into our cache */
         CopyFromUswc(cache, w2_16,
                      src, src_pitch,
-                     unaligned,
                      2*width, hblock, cpu);
 
         /* Copy from our cache to the destination */

From: Laurent Aimar <fenrir@videolan.org>
Date: Sat, 12 Jun 2010 23:41:11 +0000 (+0200)
Subject: Workaround a potential segfault when using VAAPI/DXVA2 (close #3606).
X-Git-Url: http://git.videolan.org/?p=vlc%2Fvlc-1.1.git;a=commitdiff_plain;h=f134a12a7469c003ba95177dc23e65c70a4a5a72

Workaround a potential segfault when using VAAPI/DXVA2 (close #3606).

It seems that some avcodec decoders release frames even after being flushed.

Signed-off-by: Jean-Baptiste Kempf <jb@videolan.org>
---

diff --git a/modules/codec/avcodec/video.c b/modules/codec/avcodec/video.c
index 230010d..bc37e12 100644
--- a/modules/codec/avcodec/video.c
+++ b/modules/codec/avcodec/video.c
@@ -725,7 +725,10 @@ void EndVideoDec( decoder_t *p_dec )
     if( p_sys->p_ff_pic ) av_free( p_sys->p_ff_pic );
 
     if( p_sys->p_va )
+    {
         vlc_va_Delete( p_sys->p_va );
+        p_sys->p_va = NULL;
+    }
 }
 
 /*****************************************************************************
@@ -1079,25 +1082,24 @@ static void ffmpeg_ReleaseFrameBuf( struct AVCodecContext *p_context,
     if( p_sys->p_va )
     {
         vlc_va_Release( p_sys->p_va, p_ff_pic );
-
-        /* */
-        for( int i = 0; i < 4; i++ )
-            p_ff_pic->data[i] = NULL;
     }
     else if( !p_ff_pic->opaque )
     {
-        avcodec_default_release_buffer( p_context, p_ff_pic );
+        /* We can end up here without the AVFrame being allocated by
+         * avcodec_default_get_buffer() if VA is used and the frame is
+         * released when the decoder is closed
+         */
+        if( p_ff_pic->type == FF_BUFFER_TYPE_INTERNAL )
+            avcodec_default_release_buffer( p_context, p_ff_pic );
     }
     else
     {
         picture_t *p_pic = (picture_t*)p_ff_pic->opaque;
 
         decoder_UnlinkPicture( p_dec, p_pic );
-
-        /* */
-        for( int i = 0; i < 4; i++ )
-            p_ff_pic->data[i] = NULL;
     }
+    for( int i = 0; i < 4; i++ )
+        p_ff_pic->data[i] = NULL;
 }
 
 static void ffmpeg_NextPts( decoder_t *p_dec )