Android 视频图像实时文字化

一、前言

在本篇之前，很多博客已经实现过图片文本化，但是由于渲染方式的不合理，大部分设计都做不到尽可能实时播放，本篇将在已有的基础上进行一些优化，使得视频文字化具备一定的实时性。

下图总体上视频和文字化的画面基本是实时的，上面是SurfaceView，下面是我们自定义的View类，通过实时抓帧然后实时转bitmap，做到了基本同步。

二、现状

目前很多流行的方式是修改像素的色值，这个性能差距太大，导致卡顿非常严重，无法做到实时性。当然也有通过open gl mask实现的，但是在贴图这一块我们知道，open gl只支持绘制三角、点和线，因此“文字”纹理生成还得利用Canvas实现。

但对于对帧率要求不高的需求，是不是有更好的方案呢？

三、优化方案

优化点1: 使用Shader

网上很多博客都是利用Bitmap#getPixel和Bitmap#setPixel进行，这个计算量显然太大了，就算使用open gl 也未必好，因此首先解决的问题就是使用Shader着色。

优化点2: 提前计算好单个文字所占的最大空间

显然这个原因是更加整齐的排列文字，其次也可以做到降低计算量和提高灵活度

优化点3:使用队列

对于了编解码的开发而言，使用队列不仅可以复用buffer，而且还能提高绘制性能，另外必要时可以丢帧。

基于以上三点，基本可以做到实时字符化画面，当然，我们这里是彩色的，对于灰度图的需求，可通过设置Paint的ColorMatrix实现，总之，要避免遍历修改像素了RGB。

四、关键代码

使用shader着色

 this.bitmapShader = new BitmapShader(inputBitmap, Shader.TileMode.CLAMP, Shader.TileMode.CLAMP);

 this.mCharPaint.setShader(bitmapShader);

//用下面方式清空bitmap

 boardBitmap.bitmap.eraseColor(Color.TRANSPARENT);

计算字符size

    private Rect computeMaxCharWidth(TextPaint drawerPaint, String text) {

        if (TextUtils.isEmpty(text)) {

            return null;

        }

        Rect result = new Rect(); // 文字所在区域的矩形

        Rect md = new Rect();

        for (int i = 0; i < text.length(); i++) {

            String s = text.charAt(i) + "";

            if(TextUtils.isEmpty(s)) {

                continue;

            }

            drawerPaint.getTextBounds(s, 0, 1, md);

            if (md.width() > result.width()) {
result.right = md.width();

            }

            if (md.height() > result.height()) {
result.bottom = md.height();

            }

        }

        return result;

    }

定义双队列，实现控制和享元机制

    private BitmapPool bitmapPool = new BitmapPool();
private BitmapPool recyclePool = new BitmapPool();

static class BitmapPool {
int width;
int height;
private LinkedBlockingQueue<BitmapItem> linkedBlockingQueue = new LinkedBlockingQueue<>(5);

    }

static class BitmapItem{

        Bitmap bitmap;
boolean isUsed = false;

    }

完整代码

public class WordBitmapView extends View {

    private final DisplayMetrics mDM;

    private TextPaint mCharPaint;

    private TextPaint mDrawerPaint = null;

    private Bitmap inputBitmap;

    private Rect charMxWidth = null ;

    private String text = "a1b2c3d4e5f6h7j8k9l0";

    private float textBaseline;

    private BitmapShader bitmapShader;

    public WordBitmapView(Context context) {

        this(context, null);

    }

    public WordBitmapView(Context context, AttributeSet attrs) {

        this(context, attrs, 0);

    }

    public WordBitmapView(Context context, AttributeSet attrs, int defStyleAttr) {

        super(context, attrs, defStyleAttr);
mDM = getResources().getDisplayMetrics();

        initPaint();

    }


    @Override

    protected void onMeasure(int widthMeasureSpec, int heightMeasureSpec) {

        super.onMeasure(widthMeasureSpec, heightMeasureSpec);


        int widthMode = MeasureSpec.getMode(widthMeasureSpec);

        int widthSize = MeasureSpec.getSize(widthMeasureSpec);


        if (widthMode != MeasureSpec.EXACTLY) {
widthSize = mDM.widthPixels / 2;

        }


        int heightMode = MeasureSpec.getMode(heightMeasureSpec);

        int heightSize = MeasureSpec.getSize(heightMeasureSpec);


        if (heightMode != MeasureSpec.EXACTLY) {
heightSize = widthSize / 2;

        }

        setMeasuredDimension(widthSize, heightSize);

textBaseline = getTextPaintBaseline(mDrawerPaint);

    }



    public float dp2px(float dp) {

        return TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_DIP, dp, mDM);

    }


    public float sp2px(float dp) {

        return TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_SP, dp, mDM);

    }


    @Override

    protected void onSizeChanged(int w, int h, int oldw, int oldh) {

        super.onSizeChanged(w, h, oldw, oldh);

        recyclePool.clear();

        bitmapPool.clear();

    }


    Matrix matrix = new Matrix();

    @Override

    protected void onDraw(Canvas canvas) {

        super.onDraw(canvas);

        int width = getWidth();

        int height = getHeight();

        if (width <= 1 || height <= 1) {

            return;

        }

        BitmapItem bitmapItem = bitmapPool.linkedBlockingQueue.poll();

        if (bitmapItem == null || inputBitmap == null) {

            return;

        }

        if(!bitmapItem.isUsed){

            return;

        }

        canvas.drawBitmap(bitmapItem.bitmap,matrix,mDrawerPaint);
bitmapItem.isUsed = false;

        try {

            recyclePool.linkedBlockingQueue.offer(bitmapItem,16,TimeUnit.MILLISECONDS);

        } catch (InterruptedException e) {

            e.printStackTrace();

        }

    }


    public static float getTextPaintBaseline(Paint p) {

        Paint.FontMetrics fontMetrics = p.getFontMetrics();

        return (fontMetrics.descent - fontMetrics.ascent) / 2 - fontMetrics.descent;

    }


    private Rect computeMaxCharWidth(TextPaint drawerPaint, String text) {

        if (TextUtils.isEmpty(text)) {

            return null;

        }

        Rect result = new Rect(); // 文字所在区域的矩形

        Rect md = new Rect();

        for (int i = 0; i < text.length(); i++) {

            String s = text.charAt(i) + "";

            if(TextUtils.isEmpty(s)) {

                continue;

            }

            drawerPaint.getTextBounds(s, 0, 1, md);

            if (md.width() > result.width()) {
result.right = md.width();

            }

            if (md.height() > result.height()) {
result.bottom = md.height();

            }

        }

        return result;

    }


    private BitmapPool bitmapPool = new BitmapPool();

    private BitmapPool recyclePool = new BitmapPool();


    static class BitmapPool {

        int width;

        int height;

        private LinkedBlockingQueue<BitmapItem> linkedBlockingQueue = new LinkedBlockingQueue<>(5);

        public void clear(){

            Iterator<BitmapItem> iterator = linkedBlockingQueue.iterator();

            do{

                if(!iterator.hasNext()) break;

                BitmapItem next = iterator.next();

                if(!next.bitmap.isRecycled()) {

                    next.bitmap.recycle();

                }

                iterator.remove();

            }while (true);

        }


        public int getWidth() {

            return width;

        }


        public int getHeight() {

            return height;

        }


        public void setHeight(int height) {
this.height = height;

        }


        public void setWidth(int width) {
this.width = width;

        }

    }


    class BitmapItem{

        Bitmap bitmap;

        boolean isUsed = false;

    }



   //视频图片入队

    public void queueInputBitmap(Bitmap inputBitmap) {
this.inputBitmap = inputBitmap;


        if(charMxWidth  == null){
charMxWidth = computeMaxCharWidth(mDrawerPaint,text);

        }

        if(charMxWidth == null || charMxWidth.width() == 0){

            return;

        }


        if(this.bitmapPool != null && this.inputBitmap != null){

            if(this.bitmapPool.getWidth() != this.inputBitmap.getWidth()){

                bitmapPool.clear();

                recyclePool.clear();

            }else if(this.bitmapPool.getHeight() != this.inputBitmap.getHeight()){

                bitmapPool.clear();

                recyclePool.clear();

            }

        }

        bitmapPool.setWidth(inputBitmap.getWidth());

        bitmapPool.setHeight(inputBitmap.getHeight());

        recyclePool.setWidth(inputBitmap.getWidth());

        recyclePool.setHeight(inputBitmap.getHeight());


        BitmapItem boardBitmap = recyclePool.linkedBlockingQueue.poll();

        if (boardBitmap == null && inputBitmap != null) {
boardBitmap = new BitmapItem();
boardBitmap.bitmap = Bitmap.createBitmap(inputBitmap.getWidth(), inputBitmap.getHeight(), Bitmap.Config.ARGB_8888);

        }
boardBitmap.isUsed = true;

        int bitmapWidth = inputBitmap.getWidth();

        int bitmapHeight = inputBitmap.getHeight();

        int unitWidth = (int) (charMxWidth.width()  *1.5);

        int unitHeight = charMxWidth.height()  + 2;

        int centerY = charMxWidth.centerY();

        float hLineCharNum = bitmapWidth * 1F / unitWidth;

        float vLineCharNum = bitmapHeight * 1F / unitHeight;


this.bitmapShader = new BitmapShader(inputBitmap, Shader.TileMode.CLAMP, Shader.TileMode.CLAMP);

        this.mCharPaint.setShader(bitmapShader);


        boardBitmap.bitmap.eraseColor(Color.TRANSPARENT);


        Canvas drawCanvas = new Canvas(boardBitmap.bitmap);

        int k = (int) (Math.random() * text.length());

        for (int i = 0; i < vLineCharNum; i++) {

            for (int j = 0; j < hLineCharNum; j++) {

                int length = text.length();

                int x = unitWidth * j;

                int y = centerY + i * unitHeight;

                String c = text.charAt(k % length) + "";

                drawCanvas.drawText(c, x, y + textBaseline, mCharPaint);

                k++;

            }

        }

        try {

            bitmapPool.linkedBlockingQueue.offer(boardBitmap,16, TimeUnit.MILLISECONDS);

        } catch (InterruptedException e) {

            e.printStackTrace();

        }

        postInvalidate();


    }

    private void initPaint() {

        // 实例化画笔并打开抗锯齿
mCharPaint = new TextPaint(Paint.ANTI_ALIAS_FLAG);

        mCharPaint.setAntiAlias(true);

        mCharPaint.setStyle(Paint.Style.FILL);

        mCharPaint.setStrokeCap(Paint.Cap.ROUND);
mDrawerPaint = new TextPaint(Paint.ANTI_ALIAS_FLAG);

        mDrawerPaint.setAntiAlias(true);

        mDrawerPaint.setStyle(Paint.Style.FILL);

        mDrawerPaint.setStrokeCap(Paint.Cap.ROUND);

    }



}

五、总结

Android中Shader是非常重要的工具，我们无需单独修改像素的情况下就能实现快速渲染字符，得意与Shader出色的渲染能力。另外由于时间原因，这里对字符的绘制并没有做到很精确，仅仅选了一些比较中规中列的排列，后续再继续完善吧。