Find the properties of the vector after the rotation of the canvas

EDIT: I did a video to explain what I wan't to do https://youtu.be/-6I503Urhvg

Objective: Find initial X,initial Y,final X and final Y points on canvas after rotation.

Problem: Formula is not working in my code and I need to find out what am I doing wrong.

Question:I would like to know if the formula I am using is right and how to use it properly to find objective .

Formula I am using is:

nX = x * Math.cos(-angle) - y * Math.sin(-angle)
nY = x * sin(-angle) + y * Math.cos(-angle)

The Formula seems to almost work as it closely follows the rotation of the vector. I am using little squares to see what is the position that the formula is giving me.

Formula in code:

this.nix = this.ix * Math.cos(-(rotation * (Math.PI / 180))) - this.iy * Math.sin(-(rotation * (Math.PI / 180)));
        this.niy = this.ix * Math.sin(-(rotation * (Math.PI / 180))) + this.iy * Math.cos(-(rotation * (Math.PI / 180)));
        this.nfx = this.fx * Math.cos(-(rotation * (Math.PI / 180))) - this.fy * Math.sin(-(rotation * (Math.PI / 180)));
        this.nfy = this.fx * Math.sin(-(rotation * (Math.PI / 180))) + this.fy * Math.cos(-(rotation * (Math.PI / 180)));
        ctx.fillRect(this.nix,this.niy,5,5);
        ctx.fillRect(this.nfx,this.nfy,5,5);
/*GLOBAL*/var check,lineSquare,rotation,lineRotate;
rotation = 0;
check = false;
lineRotate = false;
function lineRotateBtn(){
    lineRotate = true;
}
function setRotate(){
    var intervalo2 = setInterval(function(){rotation += 0.01;},1000/60);
}
function load() {
    var canvas = document.getElementById('box');
    var ctx = canvas.getContext('2d');

    function player(x, y, width) {
        this.x = x;
        this.y = y;
        this.width = width;

        this.vx = 3;
        this.vy = 3;

        this.up = false;
        this.down = false;
        this.left = false;
        this.right = false;

        this.drawn = function () {
            ctx.fillStyle = "black";
            ctx.fillRect(this.x, this.y, this.width, this.width);
            if (this.up) {
                this.y -= this.vy;
            }
            if (this.down) {
                this.y += this.vy;
            }
            if (this.left) {
                this.x -= this.vx;
            }
            if (this.right) {
                this.x += this.vx;
            }
        }
    }

    var players = new player(100, 100, 20);

    var keyUp, keyDown, keyLeft, keyRight;
    keyUp = 87;
    keyDown = 83;
    keyLeft = 65;
    keyRight = 68;
    window.addEventListener("keydown", checkKeyDown, false);
    function checkKeyDown(event) {
        if (event.keyCode == keyUp) {
            players.up = true;
        } else if (event.keyCode == keyDown) {
            players.down = true;
        } else if (event.keyCode == keyLeft) {
            players.left = true;
        } else if (event.keyCode == keyRight) {
            players.right = true;
        }
    }

    window.addEventListener("keyup", checkKeyUp, false);
    function checkKeyUp(event) {
        if (event.keyCode == keyUp) {
            players.up = false;
        } else if (event.keyCode == keyDown) {
            players.down = false;
        } else if (event.keyCode == keyLeft) {
            players.left = false;
        } else if (event.keyCode == keyRight) {
            players.right = false;
        }
    }

    document.onmousemove = mouseMove;
    function mouseMove(event) {
        event = event || canvas.event
        mouseX = event.pageX;
        mouseY = event.pageY;
        mouseX = mouseX - 11;
        mouseY = mouseY - 13;
        document.getElementById('test2').innerHTML = mouseX + " " + mouseY;
    }

    function line(ix, iy, fx, fy) {
        this.ix = ix;
        this.iy = iy;
        this.fx = fx;
        this.fy = fy;

        this.update = function () {
            this.nix = this.ix * Math.cos(-(rotation * (Math.PI / 180))) - this.iy * Math.sin(-(rotation * (Math.PI / 180)));
            this.niy = this.ix * Math.sin(-(rotation * (Math.PI / 180))) + this.iy * Math.cos(-(rotation * (Math.PI / 180)));
            this.nfx = this.fx * Math.cos(-(rotation * (Math.PI / 180))) - this.fy * Math.sin(-(rotation * (Math.PI / 180)));
            this.nfy = this.fx * Math.sin(-(rotation * (Math.PI / 180))) + this.fy * Math.cos(-(rotation * (Math.PI / 180)));
            ctx.fillRect(this.nix,this.niy,5,5);
            ctx.fillRect(this.nfx,this.nfy,5,5);

            var v1 = {}; //Player
            v1.vx = players.vx;
            v1.vy = players.vy;
            v1.m = Math.sqrt(v1.vx * v1.vx + v1.vy * v1.vy);
            v1.dx = v1.vx / v1.m;
            v1.dy = v1.vy / v1.m;
            var v2 = {ln: {}}; //Line;
            v2.ix = this.ix;
            v2.iy = this.iy;
            v2.fx = this.fx;
            v2.fy = this.fy;
            v2.vx = v2.fx - v2.ix;
            v2.vy = v2.fy - v2.iy;
            v2.m = Math.sqrt(v2.vx * v2.vx + v2.vy * v2.vy);v2.dx = v2.vx / v2.m;
            v2.dy = v2.vy / v2.m;
            v2.ln.vx = v2.vy;
            v2.ln.vy = -v2.vx;v2.ln.dx = v2.ln.vx / v2.m || 0;
            v2.ln.dy = v2.ln.vy / v2.m || 0;
            var v3 = {};
            v3.vx = v2.ix - players.x;v3.vy = v2.iy - players.y;
            var dp1 = v3.vx * v2.dx + v3.vy * v2.dy;
            var dp2 = v3.vx * v2.ln.dx + v3.vy * v2.ln.dy;

            if (dp1 > -v2.m && dp1 < 0) {
                if (dp2 <= 2 && dp2 >= 0) {
                    console.log('colision');
                    if (players.up && players.right){
                        players.x -= players.vx*2;
                        players.y += players.vy*2;
                    }
                    players.x -= players.vx;
                    players.y += players.vy;
                }
                if (dp2 < -1 && dp2 >= -4) {
                    console.log('colision');
                    if (players.down && players.left){
                        players.x -= players.vx*2;
                        players.y += players.vy*2;
                    }
                    players.x += players.vx;
                    players.y -= players.vy;
                }
            }
            ctx.save();
            ctx.translate(this.ix + v2.vx / 2, this.iy + v2.vy / 2);
            ctx.rotate(rotation * (Math.PI / 180));
            ctx.translate(-(this.ix + v2.vx / 2), -(this.iy + v2.vy / 2));

            ctx.beginPath();
            ctx.moveTo(this.ix, this.iy);
            ctx.lineTo(this.fx, this.fy);
            ctx.stroke();

            ctx.restore();

            document.getElementById('test1').innerHTML = "dotProduct1: " + dp1 + "v2.m: " + (-v2.m) + "<br>" + dp2;
        }
    }
    var linez = new line(300,300,400,400);

    var intervalo = setInterval(animation, 1000 / 60);
    function animation() {
        check = true;
        ctx.clearRect(0, 0, canvas.width, canvas.height);
        players.drawn();
        linez.update();
    }
}
#box {
    border: 1px solid black;
}
#button {
    border: none;
    background-color: gray;
    width: 70;
    height: 50;
}
canvas {
-webkit-touch-callout: none;
-webkit-user-select: none;
-khtml-user-select: none;
-moz-user-select: none;
-ms-user-select: none;
user-select: none;
}
html {
    background-color: white;
    color: black;
    font-family: courier new;
}
<html>
    <head>
        <meta charset="utf-8">
        <link rel="stylesheet" type="text/css" href="css/estilo.css">
        <script src="js/main.js"></script>
    </head>
    <body onload="load()">
        <canvas id="box" width="1330" height="500"></canvas>
        <button onclick="setRotate()">Rotate</button>
        <h2 id="test1"></h2>
        <h2 id="test2"></h2>
    </body>
</html>

According to video, your rotation is about left top window corner - coordinate origin.

To make rotation about arbitrary point cx, cy , you have to use these formulas:

NX = CX + (X0-CX) * Cos(Theta) - (Y0-CY) * Sin(Theta)
 NY = CY + (X0-CX) * Sin(Theta) + (Y0-CY) * Cos(Theta)