我的音視頻技術(shù)路線

try {
            mCamera = Camera.open(mCurrentCamera);
            Camera.Parameters params = mCamera.getParameters();
            List<Camera.Size> previewSizes = params.getSupportedPreviewSizes();

            Camera.Size preViewSize = previewSizes.get(previewSizes.size() > 4 ? previewSizes.size() - 4 : 0);
            params.setPreviewSize(preViewSize.width, preViewSize.height);
            mPreviewHeight = preViewSize.height;
            mPreviewWidth = preViewSize.width;
            Log.e(TAG, "preViewSize->width: " + preViewSize.width + ", preViewSize->height: " + preViewSize.height);

            params.setPictureFormat(ImageFormat.JPEG);
            params.setJpegQuality(100);
            //是否開啟閃光
            List<String> flashModes = params.getSupportedFlashModes();
            if (flashModes != null && flashModes.contains(Camera.Parameters.FLASH_MODE_OFF)) {
                params.setFlashMode(Camera.Parameters.FLASH_MODE_OFF);
            }

            mCamera.setPreviewCallback(this);
            mCamera.setDisplayOrientation(PORTRAIT_MODE);
            mCamera.setParameters(params);
            if(!Consts.SHOW_CAMERA) {
                LogUtils.log_main_step("相機(jī)開始preview");
                mCamera.startPreview();
            }

            LogUtils.logd(TAG, "camera init finish.....");
        } catch (Exception e) {
            LogUtils.loge(TAG, e.getMessage());
            e.printStackTrace();
        }

@Override
    public void onPreviewFrame(byte[] data, Camera camera) {
        Camera.CameraInfo mCameraInfo = new Camera.CameraInfo();
        //如果使用前置攝像頭，請(qǐng)注意顯示的圖像與幀圖像左右對(duì)稱，需處理坐標(biāo)
        boolean frontCamera = (mCurrentCamera == Camera.CameraInfo.CAMERA_FACING_FRONT);

        //獲取重力傳感器返回的方向
        int dir = getDirection();

        int rotate = (dir ^ 1);
        //Log.e("stRotateCamera  ", (dir ^ 1) + " rotate result");

        //在使用后置攝像頭，且傳感器方向?yàn)?或2時(shí)，后置攝像頭與前置orentation相反
        if (!frontCamera && dir == 0) {
            dir = 2;
        } else if (!frontCamera && dir == 2) {
            dir = 0;
        }
        dir = (dir ^ 2);
        //.....
        process(data)
    }

CameraManager cameraManager = (CameraManager)mContext.getSystemService(Context.CAMERA_SERVICE);
//可以拿到所以的相機(jī)列表，比如Wide，Tele，Macro，Tele2x，Tele4x等等
String[] cameraIdList = cameraManager.getCameraIdList();
//根據(jù)Camera ID拿到對(duì)應(yīng)設(shè)備支持的能力
CameraCharacteristics cameraCharacteristics =
  cameraManager.getCameraCharacteristics(cameraIdStr);
//比如用這個(gè)去判斷支持的最小Focus distance
Float focusDistance = mCharacteristics.get(CameraCharacteristics.LENS_INFO_MINIMUM_FOCUS_DISTANCE);

String cameraIdStr = String.valueOf(mCameraId);
            cameraManager.openCamera(cameraIdStr, mCameraStateCallback, mMainHandler);
//
private CameraDevice.StateCallback mCameraStateCallback = new CameraDevice.StateCallback() {

        @Override
        public void onOpened(@NonNull CameraDevice camera) {
            synchronized (SnapCamera.this) {
                mCameraDevice = camera;
            }
            if (mStatusListener != null) {
                mStatusListener.onCameraOpened();
            }
        }

        @Override
        public void onDisconnected(@NonNull CameraDevice camera) {
            Log.w(TAG, "onDisconnected");
            // fail-safe: make sure resources get released
            release();
        }

        @Override
        public void onError(@NonNull CameraDevice camera, int error) {
            Log.e(TAG, "onError: " + error);
            // fail-safe: make sure resources get released
            release();
        }
    };      

public static final int TEMPLATE_MANUAL = 6;
public static final int TEMPLATE_PREVIEW = 1; 
public static final int TEMPLATE_RECORD = 3;
public static final int TEMPLATE_STILL_CAPTURE = 2;
public static final int TEMPLATE_VIDEO_SNAPSHOT = 4;
public static final int TEMPLATE_ZERO_SHUTTER_LAG = 5;
//創(chuàng)建的代碼
mPreviewRequestBuilder = mCameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);

//這里是更加專業(yè)一點(diǎn)的用法，可以不看，就是拿到RequestBuilder之后我們可以去下發(fā)各種TAG，如下設(shè)置AE/AWB鎖
CaptureRequestBuilder.applyAELock(request, mConfigs.isAELocked());
CaptureRequestBuilder.applyAWBLock(request, mConfigs.isAWBLocked());

//SurfaceTexture的方式，用于預(yù)覽
mSurfaceTexture = new SurfaceTexture(false);
mSurfaceTexture.setDefaultBufferSize(optimalSize.width, optimalSize.height);
mPreviewSurface = new Surface(mSurfaceTexture);
//ImageReader 的方式, Format可以是YUV、RAW，DEPTH等
mPhotoImageReader = ImageReader.newInstance(size.getWidth(), size.getHeight(),
                ImageFormat.JPEG, /* maxImages */ 2);
mPhotoImageReader.setOnImageAvailableListener(mPhotoAvailableListener, mCameraHandler);
//創(chuàng)建session
List<Surface> surfaces = Arrays.asList(mPreviewSurface, mPhotoImageReader.getSurface());
mCameraDevice.createCaptureSession(surfaces, mSessionCallback, mCameraHandler);

mCameraDevice.createCaptureSession(surfaces, mSessionCallback, mCameraHandler);
private CameraCaptureSession.StateCallback
            mSessionCallback = new CameraCaptureSession.StateCallback() {

        @Override
        public void onConfigured(@NonNull CameraCaptureSession session) {
            synchronized (SnapCamera.this) {
                if (mCameraDevice == null) {
                    Log.e(TAG, "onConfigured: CameraDevice was already closed.");
                    session.close();
                    return;
                }
                mCaptureSession = session;
            }
            startPreview();
            capture();
        }

        @Override
        public void onConfigureFailed(@NonNull CameraCaptureSession session) {
            Log.e(TAG, "sessionCb: onConfigureFailed");
        }
    };

/*發(fā)起請(qǐng)求，后續(xù)相機(jī)開始往surface輸出數(shù)據(jù)，這個(gè)可以在onConfigured里面調(diào)用，如上面的代碼                  startPreview();
capture();*/
mCaptureSession.setRepeatingRequest(mPreviewRequestBuilder.build(),
                    mCaptureCallback, mCameraHandler);
mCaptureSession.capture(requestBuilder.build(), mCaptureCallback, mCameraHandler);

// 這個(gè)是針對(duì)120/240/960Fps recording
mCaptureSession.setRepeatingBurst(requestList, mCaptureCallback, mCameraHandler);
mCaptureSession.captureBurst(requestList, listener, handler);

mCaptureSession.abortCaptures();
mCaptureSession.stopRepeating();

//每一個(gè)等級(jí)啥意思自己搜吧
INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY
INFO_SUPPORTED_HARDWARE_LEVEL_LIMITED
INFO_SUPPORTED_HARDWARE_LEVEL_FULL
INFO_SUPPORTED_HARDWARE_LEVEL_3
INFO_SUPPORTED_HARDWARE_LEVEL_EXTERNAL

//預(yù)覽
PreviewConfig config = new PreviewConfig.Builder().build();
Preview preview = new Preview(config);

 preview.setOnPreviewOutputUpdateListener(
     new Preview.OnPreviewOutputUpdateListener() {
         @Override
         public void onUpdated(Preview.PreviewOutput previewOutput) {
             // Your code here. For example, use previewOutput.getSurfaceTexture()
             // and post to a GL renderer.
         };
 });

 CameraX.bindToLifecycle((LifecycleOwner) this, preview);

//拍照
ImageCaptureConfig config =
new ImageCaptureConfig.Builder()
        .setTargetRotation(getWindowManager().getDefaultDisplay().getRotation())
        .build();

ImagesCapture imageCapture = new ImageCapture(config);
CameraX.bindToLifecycle((LifecycleOwner) this, imageCapture, imageAnalysis, preview);

ImageCapture.Metadata metadata = new ImageCapture.Metadata();
        metadata.isReversedHorizontal = mCameraLensFacing == LensFacing.FRONT;
imageCapture .takePicture(saveLocation, metadata, executor, OnImageSavedListener);

local intpx = 220      --柱子左右間隔
local intpy = 550      --柱子中間縫隙寬度
local range_y = {0.3, 0.7}      --柱子縫隙中央隨機(jī)范圍，{0.3,0.7}代表縫隙可能在屏幕高度30%-70%的位置隨機(jī)出現(xiàn)
local sp = {0.5,1.0}         --速度初始和最終速度，{0.5,1.0}代表初始速度為1秒走過(guò)0.5個(gè)屏幕，最終速度為1秒走過(guò)1個(gè)屏幕
local ptime = 3           --準(zhǔn)備時(shí)間3s
local range_s = {5,10}  --分?jǐn)?shù)分段，{5,10}代表0-5第一段，6-10第二段，10以上第三段；10以上未碰撞第四段
local smul = 0.8        --字號(hào)倍率

  //.....此次省略N行

//兩個(gè)碰撞檢測(cè)函數(shù)
//u=1,2 代碼上面和下面柱子
local function rectCollision(i,u)
    local K_s_x = sub.x
    local K_s_y = sub.y * ratio
    local K_rect_x = pillar[i].x
    local K_rect_y = 0


    local l = K_rect_x - r 
    local r = K_rect_x + r 
    local t = 0
    local b = 0
    if u == 1 then
        t = -1
        b = (pillar[i].y - interval_y / 2) * ratio
    else
        t = (pillar[i].y + interval_y / 2) * ratio
        b = 2
    end

    local closestP_x = 0.0
    local closestP_y = 0.0

    closestP_x = clamp(K_s_x, l , r)
    closestP_y = clamp(K_s_y, t , b)
    local dist = distance(closestP_x, closestP_y, K_s_x, K_s_y)
    if dist <= sub.r * 0.9 then 
        return true
    else
        return false
    end
    return false
end

local function circleCollision(i,u)
    local K_s_x = sub.x
    local K_s_y = sub.y * ratio
    local K_cir_x = pillar[i].x
    local K_cir_y = 0

    if u == 1 then
        K_cir_y = (pillar[i].y - interval_y / 2) * ratio
    else
        K_cir_y = (pillar[i].y + interval_y / 2) * ratio
    end
    local dist = distance(K_cir_x, K_cir_y, K_s_x, K_s_y)
    if dist <= sub.r * 0.9 + r then 
        return true
    else
        return false
    end
    return false
end

handleTimerEvent = function(this, timerId, milliSeconds)
        if timerId == timer_ID_Fast and gaming then
            if init_state ~= 0 then
                return true
            end

            timeThis = getTime(this)
            timeDelta = getDiffTime(timeLast, timeThis)
            timeCumu = timeCumu + timeDelta
            timeLast = timeThis

            local speed = sp[1] + clamp(timeCumu / 14, 0,1) * (sp[2] - sp[1])

            if timeCumu > 14 then
                gaming = false
                Sticker2DV3.playClip(this, feature_0.folder, feature_0.entity[1], feature_0.clip[1], 0)

                CommonFunc.setFeatureEnabled(this, feature_t.folder, true)
                if score >= 10 then
                    realTimeFunc.initRealTime(this, feature_t.folder, true, feature_t.realTimeParams[1])
                else
                    realTimeFunc.initRealTime(this, feature_t.folder, true, feature_t.realTimeParams[2])
                end
                realTimeFunc.setRealTime(this, feature_t.folder, "u_data", score)

            end

            for i = 1,5 do
                pillar[i].x = pillar[i].x - speed * timeDelta
                if pillar[i].x < -r then
                    pillar[i].x = pillar[i].x + interval_x * 5
                    pillar[i].y = range_y[1] + math.random() * (range_y[2] - range_y[1])
                    pillar[i].counted = false
                end
                if pillar[i].x < sub.x and not pillar[i].counted then
                    pillar[i].counted = true
                    score = score + 1

                end

                local cy = pillar[i].y - (interval_y / 2 - r / ratio) - 0.5
                local cx = pillar[i].x
                set4Vtx(this, feature_3.folder, feature_3.entity[1], feature_3.clip[2*i-1], cx , cy, 2 * r, 1.0 , 0.0, 1.0, ratio, 0.0, 0.0)

                cy = pillar[i].y + (interval_y / 2 - r / ratio) + 0.5
                //更新實(shí)體坐標(biāo)
                set4Vtx(this, feature_3.folder, feature_3.entity[1], feature_3.clip[2*i], cx , cy, 2 * r, 1.0 , 0.0, 1.0, ratio, 0.0, 0.0)
            end

            if noseY ~= 0 then
                sub.y = noseY
            end
            if (sub.y ~= 0 or lastY ~= 0) then
                sub.a = sub.a * 0.8 + (sub.y - lastY) / timeDelta * 0.2
            else
                sub.a = sub.a * 0.8
            end

            set4Vtx(this, feature_2.folder, feature_2.entity[1], feature_2.clip[1], sub.x , sub.y, 2 * sub.r, 2 * sub.r / ratio / 1.1 , sub.a, 1.0, ratio, 0.0, 0.0)

                        //對(duì)所有柱子遍歷做碰撞檢測(cè)
            for i = 1,5 do
                if rectCollision(i,1) or rectCollision(i,2) or circleCollision(i,1) or circleCollision(i,2) then

                    gaming = false
                    if score <= range_s[1] then
                        Sticker2DV3.playClip(this, feature_0.folder, feature_0.entity[1], feature_0.clip[4], 0)
                    elseif score <= range_s[2] then
                        Sticker2DV3.playClip(this, feature_0.folder, feature_0.entity[1], feature_0.clip[3], 0)
                    else
                        Sticker2DV3.playClip(this, feature_0.folder, feature_0.entity[1], feature_0.clip[2], 0)
                    end

                    CommonFunc.setFeatureEnabled(this, feature_t.folder, true)
                    if score >= 10 then
                        realTimeFunc.initRealTime(this, feature_t.folder, true, feature_t.realTimeParams[1])
                    else
                        realTimeFunc.initRealTime(this, feature_t.folder, true, feature_t.realTimeParams[2])
                    end
                    realTimeFunc.setRealTime(this, feature_t.folder, "u_data", score)
                end

            end

            lastY = sub.y
        end

        return true
    end,


    handleRecodeVedioEvent = function (this, eventCode)
        if (init_state ~= 0) then
            return true
        end
        if (eventCode == 1) then
            timeLast = getTime(this)
            timeCumu = 0

            score = 0
            gaming = true
            pillar = 
            {
                {
                    x = start_x,
                    y = range_y[1] + math.random() * (range_y[2] - range_y[1]),
                    counted = false
                },
                {
                    x = start_x + interval_x,
                    y = range_y[1] + math.random() * (range_y[2] - range_y[1]),
                    counted = false
                },
                {
                    x = start_x + interval_x * 2,
                    y = range_y[1] + math.random() * (range_y[2] - range_y[1]),
                    counted = false
                },
                {
                    x = start_x + interval_x * 3,
                    y = range_y[1] + math.random() * (range_y[2] - range_y[1]),
                    counted = false
                },
                {
                    x = start_x + interval_x * 4,
                    y = range_y[1] + math.random() * (range_y[2] - range_y[1]),
                    counted = false
                },
            }

            sub = 
            {
                r = 0.105,
                x = 0.24,
                y = 0.5,
                sy = 0.0,
                a = 0
            }
            lastY = noseY
            for i = 1,10 do
                Sticker2DV3.playClip(this, feature_3.folder, feature_3.entity[1], feature_3.clip[i], 0)
                set4Vtx(this, feature_3.folder, feature_3.entity[1], feature_3.clip[i], -2 , -2, 0, 0 , 0.0, 0.0, ratio, 0.0, 0.0)
            end
            for i = 1,4 do
                Sticker2DV3.stopClip(this, feature_0.folder, feature_0.entity[1], feature_0.clip[i])
            end
            set4Vtx(this, feature_2.folder, feature_2.entity[1], feature_2.clip[1], -2 , -2, 0, 0 , 0, 0, ratio, 0.0, 0.0)
            CommonFunc.setFeatureEnabled(this, feature_t.folder, false)
        end
        return true
    end,
    }

"clipname1": {
        "alphaFactor": 1.0,
        "blendmode": 0,
        "fps": 16,
        "height": 801,
        "textureIdx": {
          "idx": [
            0
          ],
          "type": "image"
        },
        "transformParams": {
          "position": {
            "point0": {
              "anchor": [
                0.0,
                0.5
              ],
              "point": [
                {
                  "idx": "topright",
                  "relationRef": 0,
                  "relationType": "foreground",
                  "weight": 0.5635420937542709
                },
                {
                  "idx": "bottomleft",
                  "relationRef": 0,
                  "relationType": "foreground",
                  "weight": -0.0793309886223863
                }
              ]
            },
            "point1": {
              "anchor": [
                1.0,
                0.5
              ],
              "point": [
                {
                  "idx": "topright",
                  "relationRef": 0,
                  "relationType": "foreground",
                  "weight": 0.7854868849874516
                },
                {
                  "idx": "bottomleft",
                  "relationRef": 0,
                  "relationType": "foreground",
                  "weight": -0.0793309886223863
                }
              ]
            }
          },
          "relation": {
            "foreground": 1
          },
          "relationIndex": [
            0
          ],
          "relationRefOrder": 0,
          "rotationtype": 1,
          "scale": {
            "scaleY": {
              "factor": 1.0
            }
          }
        },

        ....

void main() {

    vec2 uv = gl_FragCoord.st/u_resolution;
    uv.y = 1.0 - uv.y;

    vec4 originColor = texture2D(u_texture, uv);
    vec4 texel = texture2D(u_texture, uv);
    vec3 bbTexel = texture2D(u_blowoutTex, uv).rgb;
        //256x1
    texel.r = texture2D(u_overlayTex, vec2(bbTexel.r, texel.r)).r;
    texel.g = texture2D(u_overlayTex, vec2(bbTexel.g, texel.g)).g;
    texel.b = clamp(texture2D(u_overlayTex, vec2(bbTexel.b, texel.b)).b, 0.1, 0.9);
        //256x3
    vec4 mapped;
    mapped.r = texture2D(u_mapTex, vec2(texel.r, .25)).r;
    mapped.g = texture2D(u_mapTex, vec2(texel.g, .5)).g;
    mapped.b = texture2D(u_mapTex, vec2(texel.b, 0.1)).b;
    mapped.a = 1.0;

    mapped.rgb = mix(originColor.rgb, mapped.rgb, 1.0);

    gl_FragColor = mapped;


}

//這沒(méi)啥說(shuō)的，都是很基本的
void main() {

    vec2 uv = gl_FragCoord.st/u_resolution;
    uv.y = 1.0 - uv.y;
    lowp vec3 textureColor = texture2D(u_texture, uv).rgb;

    textureColor = clamp((textureColor - vec3(u_levelBlack, u_levelBlack, u_levelBlack)) * u_levelRangeInv, 0.0, 1.0);
    textureColor.r = texture2D(u_grayTexture, vec2(textureColor.r, 0.5)).r;
    textureColor.g = texture2D(u_grayTexture, vec2(textureColor.g, 0.5)).g;
    textureColor.b = texture2D(u_grayTexture, vec2(textureColor.b, 0.5)).b;

    mediump float blueColor = textureColor.b * 15.0;

    mediump vec2 quad1;
    quad1.y = floor(blueColor / 4.0);
    quad1.x = floor(blueColor) - (quad1.y * 4.0);

    mediump vec2 quad2;
    quad2.y = floor(ceil(blueColor) / 4.0);
    quad2.x = ceil(blueColor) - (quad2.y * 4.0);

    highp vec2 texPos1;
    texPos1.x = (quad1.x * 0.25) + 0.5 / 64.0 + ((0.25 - 1.0 / 64.0) * textureColor.r);
    texPos1.y = (quad1.y * 0.25) + 0.5 / 64.0 + ((0.25 - 1.0 / 64.0) * textureColor.g);

    highp vec2 texPos2;
    texPos2.x = (quad2.x * 0.25) + 0.5 / 64.0 + ((0.25 - 1.0 / 64.0) * textureColor.r);
    texPos2.y = (quad2.y * 0.25) + 0.5 / 64.0 + ((0.25 - 1.0 / 64.0) * textureColor.g);

    lowp vec4 newColor1 = texture2D(u_lookupTexture, texPos1);
    lowp vec4 newColor2 = texture2D(u_lookupTexture, texPos2);

    lowp vec3 newColor = mix(newColor1.rgb, newColor2.rgb, fract(blueColor));

    textureColor = mix(textureColor, newColor, u_strength);

    gl_FragColor = vec4(textureColor, 1.0); 

}

vec4 BilateralFilter(vec2 uv) {
    float i = uv.x;
    float j = uv.y;
    float sigmaSSquare = 2.0 * SigmaS * SigmaS;
    float sigmaRSquare = 2.0 * SigmaR * SigmaR;
    vec3 centerColor = texture2D(u_texture, uv).rgb;
    float centerGray = Luminance(centerColor);
    vec3 sum_up, sum_down;
    for(int k = -u_radius; k <= u_radius; k++) {
        for(int l = -u_radius; l <= u_radius; l++) {
            vec2 uv_new = uv + vec2(k, l)/u_resolution;
            vec3 curColor = texture2D(u_texture, uv_new).rgb;
            float curGray = Luminance(curColor);
            vec3 deltaColor = curyolor - centerColor;
            float len = dot(deltaColor, deltaColor);
            float exponent = -((i-k)*(i-k)+(j-l)*(j-l))/sigmaSSquare - len/sigmaRSquare;
            float weight = exp(exponent);
            sum_up += curColor * weight;
            sum_down += weight;
        }
    }
    vec3 color = sum_up / sum_down;
    return vec4(color, 1.0);
}
vec4 SurfaceFilter(vec2 uv) {
    vec3 centerColor = texture2D(u_texture, uv).rgb;
    vec3 sum_up, sum_down;
    for(int k = -u_radius; k <= u_radius; k++) {
        for(int l = -u_radius; l <= u_radius; l++) {
            vec2 uv_new = uv + vec2(k, l)/u_resolution;
            vec3 curColor = texture2D(u_texture, uv_new).rgb;
            vec3 weight = CalculateWeight(curColor, centerColor);
            sum_up += weight * curColor;
            sum_down += weight;
        }
    }
    return vec4(sum_up / sum_down, 1.0);    
}

vec4 gaussian(vec2 uv, bool horizontalPass) {
      float numBlurPixelsPerSide = float(blurSize / 2); 
vec2 blurMultiplyVec = 0 < horizontalPass ? vec2(1.0, 0.0) : vec2(0.0, 1.0);

 //高斯函數(shù)
 vec3 incrementalGaussian;
 incrementalGaussian.x = 1.0 / (sqrt(2.0 * pi) * sigma);
 incrementalGaussian.y = exp(-0.5 / (sigma * sigma));
 incrementalGaussian.z = incrementalGaussian.y * incrementalGaussian.y;

 vec4 avgValue = vec4(0.0, 0.0, 0.0, 0.0);
 float coefficientSum = 0.0;

 avgValue += texture2D(texture, vertTexCoord.st) * incrementalGaussian.x;
 coefficientSum += incrementalGaussian.x;
 incrementalGaussian.xy *= incrementalGaussian.yz;

 for (float i = 1.0; i <= numBlurPixelsPerSide; i++) { 
   avgValue += texture2D(texture, vertTexCoord.st - i * texOffset * 
                         blurMultiplyVec) * incrementalGaussian.x;         
   avgValue += texture2D(texture, vertTexCoord.st + i * texOffset * 
                         blurMultiplyVec) * incrementalGaussian.x;         
   coefficientSum += 2.0 * incrementalGaussian.x;
   incrementalGaussian.xy *= incrementalGaussian.yz;
 }

 return avgValue / coefficientSum;```

// 曲線形變處理
vec2 curveWarp(vec2 textureCoord, vec2 originPosition, vec2 targetPosition, float radius)
{
    vec2 offset = vec2(0.0);
    vec2 result = vec2(0.0);

    vec2 direction = targetPosition - originPosition;

    float infect = distance(textureCoord, originPosition)/radius;

    infect = 1.0 - infect;
    infect = clamp(infect, 0.0, 1.0);
    offset = direction * infect;

    result = textureCoord - offset;

    return result;
}

//scale center
vec2 scale_uv = vec2(0.5 + (tc.x - 0.5) / scaleU , 0.5 + (tc.y - 0.5) / scaleV );

//rotate
vec2 rotateUV(vec2 uv, float rotation, vec2 mid)
{
  float ratio = (resolution.x / resolution.y);
  float s = sin ( rotation );
  float c = cos ( rotation );
  mat2 rotationMatrix = mat2( c, -s, s, c);
  vec2 coord = vec2((uv.x - mid.x) * ratio ,(uv.y -mid.y)*1.0);
  vec2 scaled = rotationMatrix * coord;
  return vec2(scaled.x / ratio + mid.x,scaled.y + mid.y);
}

//translate
vec2 translate_uv = vec2(0.5 + (tc.x - 0.5) / scaleU , 0.5 + (tc.y - 0.5) / scaleV );

float rand(vec2 uv){
  return fract(sin(dot(uv.xy ,vec2(12.9898,78.233))) * 43758.5453);
}
vec4 rotation_blur(vec2 tc) {
  angle = angle * PI_ROTATION / 180.0;
  vec2 uv = tc;
  float uv_random = rand(uv);
  vec4 sum_color = vec4(0.0);
  for(float i = 0.0; i < samples; i++) {
    float percent = (i + uv_random) / samples;
    float real_angle = angle + percent * strength;
    real_angle = mod(real_angle, PI_ROTATION);
    vec2 uv_rotation = rotateUV(uv, real_angle, center);
    sum_color += INPUT(fract(uv_rotation));
  }
  return sum_color / samples;
}

//xxxx

precision highp float;
varying vec2 tc;

#define FUNCNAME eff0
uniform sampler2D eff0_tex;
vec4 FUNCNAME(vec2 tc) {
    return texture2D(eff0_tex, vec2(tc.x,1.0-tc.y));
}
#undef PREFIX
#undef FUNCNAME

#define INPUT eff0

#define FUNCNAME eff1
uniform float eff1_strength;
uniform sampler2D eff1_lut;
vec4 FUNCNAME(vec2 tc) { 
    float strength = eff1_strength;
    lowp vec4 textureColor = INPUT(tc); 
    mediump float blueColor = textureColor.b * 63.0;
    mediump vec2 quad1; quad1.y = floor(floor(blueColor) / 8.0); 
    quad1.x = floor(blueColor) - (quad1.y * 8.0); 
    mediump vec2 quad2; 
    quad2.y = floor(ceil(blueColor) / 8.0); 
    quad2.x = ceil(blueColor) - (quad2.y * 8.0); 
    highp vec2 texPos1; 
    texPos1.x = (quad1.x * 0.125) + 0.5/512.0 + ((0.125 - 1.0/512.0) * textureColor.r); 
    texPos1.y = (quad1.y * 0.125) + 0.5/512.0 + ((0.125 - 1.0/512.0) * textureColor.g);
    highp vec2 texPos2; 
    texPos2.x = (quad2.x * 0.125) + 0.5/512.0 + ((0.125 - 1.0/512.0) * textureColor.r);
    texPos2.y = (quad2.y * 0.125) + 0.5/512.0 + ((0.125 - 1.0/512.0) * textureColor.g);
    lowp vec4 newColor1 = texture2D(eff1_lut, texPos1); 
    lowp vec4 newColor2 = texture2D(eff1_lut, texPos2); 
    lowp vec4 newColor = mix(newColor1, newColor2, fract(blueColor)); 
    return mix(textureColor, vec4(newColor.rgb, textureColor.w), strength);
} 
#undef PREFIX
#undef FUNCNAME
#undef INPUT

#define INPUT eff1
void main()
{
    gl_FragColor = INPUT(tc);
}

while (!m_sawOutputEOS) {
            if (!m_sawInputEOS) {
                // Feed more data to the decoder
                final int inputBufIndex = m_decoder.dequeueInputBuffer(TIMEOUT_USEC);
                if (inputBufIndex >= 0) {
                    ByteBuffer inputBuf = m_decoderInputBuffers[inputBufIndex];
                    // Read the sample data into the ByteBuffer. This neither
                    // respects nor
                    // updates inputBuf's position, limit, etc.
                    final int chunkSize = m_extractor.readSampleData(inputBuf, 0);

                    if (m_verbose)
                        Log.d(TAG, "input packet length: " + chunkSize + " time stamp: " + m_extractor.getSampleTime());

                    if (chunkSize < 0) {
                        // End of stream -- send empty frame with EOS flag set.
                        m_decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
                        m_sawInputEOS = true;
                        if (m_verbose)
                            Log.d(TAG, "Sent input EOS");
                    } else {
                        if (m_extractor.getSampleTrackIndex() != m_videoTrackIndex) {
                            Log.w(TAG, "WEIRD: got sample from track " + m_extractor.getSampleTrackIndex()
                                    + ", expected " + m_videoTrackIndex);
                        }

                        long presentationTimeUs = m_extractor.getSampleTime();

                        m_decoder.queueInputBuffer(inputBufIndex, 0, chunkSize, presentationTimeUs, 0);
                        if (m_verbose)
                            Log.d(TAG,
                                    "Submitted frame to decoder input buffer " + inputBufIndex + ", size=" + chunkSize);

                        m_inputBufferQueued = true;
                        ++m_pendingInputFrameCount;
                        if (m_verbose)
                            Log.d(TAG, "Pending input frame count increased: " + m_pendingInputFrameCount);

                        m_extractor.advance();
                        m_extractorInOriginalState = false;
                    }
                } else {
                    if (m_verbose)
                        Log.d(TAG, "Input buffer not available");
                }
            }
  final int decoderStatus = m_decoder.dequeueOutputBuffer(m_bufferInfo, dequeueTimeoutUs);
    if (decoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
                // No output available yet
                if (m_verbose)
                    Log.d(TAG, "No output from decoder available");
            } else if (decoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
                // Not important for us, since we're using Surface
                if (m_verbose)
                    Log.d(TAG, "Decoder output buffers changed");
            } else if (decoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
                MediaFormat newFormat = m_decoder.getOutputFormat();
                if (m_verbose)
                    Log.d(TAG, "Decoder output format changed: " + newFormat);
            } else if (decoderStatus < 0) {
                Log.e(TAG, "Unexpected result from decoder.dequeueOutputBuffer: " + decoderStatus);
                return ERROR_FAIL;
            } else {
            ///....
        m_decoder.releaseOutputBuffer(decoderStatus, doRender);
        }
    }

if (avformat_open_input(&mFormatContext, path.c_str(), NULL, NULL) != 0) {
        mlt_log_error(mProducer, "Could not open input file: %s", path.c_str());
        goto error;
    }

if (avformat_find_stream_info(mFormatContext, NULL) < 0) {
  mlt_log_error(mProducer, "Could not find stream information");
  goto error;
}

mVideoStreamIndex = findPreferedVideoStream();//遍歷track找到視頻軌
mVideoStream = mFormatContext->streams[mVideoStreamIndex];

//然后就可以得到一些列參數(shù)，width，height，fps，fromat


//如果是硬解
//根據(jù)不同的編碼器，找到對(duì)應(yīng)的解碼器，如H264，H265等
mVideoCodec = avcodec_find_decoder_by_name("h264_mediacodec");;

//如果是軟解
mVideoCodec = avcodec_find_decoder(mVideoStream->codecpar->codec_id);

//open
avcodec_open2(mVideoCodecContext, mVideoCodec, NULL)
//就可以開始解碼了

do {
        if (!mPacket) {
            ret = getVideoPacket();
            if (ret < 0 && ret != AVERROR_EOF) {
                break;
            }
        }
                //這里面送包的時(shí)候有很多細(xì)節(jié)
            int ret = avcodec_send_packet(mVideoCodecContext, flush ? NULL : mPacket);
            if (ret >= 0)
            freeVideoPacket();
        else if (ret < 0 && ret != AVERROR_EOF && ret != AVERROR(EAGAIN)) {
            break;
        }
        ret = avcodec_receive_frame(mVideoCodecContext, mFrame);
        if (ret >= 0) {

            ret = DECODER_FFMPEG_SUCCESS;
            //計(jì)算pts
            mCurrentPos = getTime(mFrame->pts == AV_NOPTS_VALUE ? mFrame->pkt_dts : mFrame->pts);
            pts = mCurrentPos;

            break;
        }

        ++count;
    } while (ret < DECODER_FFMPEG_SUCCESS && count < DECODER_TRY_ATTEMPTS);

//后面就是針對(duì)不同的格式，接入數(shù)據(jù)，
//還可以通過(guò)swscaleFrame進(jìn)行轉(zhuǎn)碼，轉(zhuǎn)成你想要的格式

//然后就是copy AVFrame里面的數(shù)據(jù)啦

+--------+   +------+   +--------+
|Producer|-->|Filter|-->|Consumer|
+--------+   +------+   +--------+